third_party/cython/src/Cython/Build/Dependencies.py

   1 import cython
   2 from Cython import __version__
   3
   4 import re, os, sys, time
   5 try:
   6     from glob import iglob
   7 except ImportError:
   8     # Py2.4
   9     from glob import glob as iglob
  10
  11 try:
  12     import gzip
  13     gzip_open = gzip.open
  14     gzip_ext = '.gz'
  15 except ImportError:
  16     gzip_open = open
  17     gzip_ext = ''
  18 import shutil
  19 import subprocess
  20
  21 try:
  22     import hashlib
  23 except ImportError:
  24     import md5 as hashlib
  25
  26 try:
  27     from io import open as io_open
  28 except ImportError:
  29     from codecs import open as io_open
  30
  31 try:
  32     from os.path import relpath as _relpath
  33 except ImportError:
  34     # Py<2.6
  35     def _relpath(path, start=os.path.curdir):
  36         if not path:
  37             raise ValueError("no path specified")
  38         start_list = os.path.abspath(start).split(os.path.sep)
  39         path_list = os.path.abspath(path).split(os.path.sep)
  40         i = len(os.path.commonprefix([start_list, path_list]))
  41         rel_list = [os.path.pardir] * (len(start_list)-i) + path_list[i:]
  42         if not rel_list:
  43             return os.path.curdir
  44         return os.path.join(*rel_list)
  45
  46
  47 from distutils.extension import Extension
  48
  49 from Cython import Utils
  50 from Cython.Utils import cached_function, cached_method, path_exists, find_root_package_dir
  51 from Cython.Compiler.Main import Context, CompilationOptions, default_options
  52
  53 join_path = cached_function(os.path.join)
  54
  55 if sys.version_info[0] < 3:
  56     # stupid Py2 distutils enforces str type in list of sources
  57     _fs_encoding = sys.getfilesystemencoding()
  58     if _fs_encoding is None:
  59         _fs_encoding = sys.getdefaultencoding()
  60     def encode_filename_in_py2(filename):
  61         if isinstance(filename, unicode):
  62             return filename.encode(_fs_encoding)
  63         return filename
  64 else:
  65     def encode_filename_in_py2(filename):
  66         return filename
  67     basestring = str
  68
  69 def extended_iglob(pattern):
  70     if '**/' in pattern:
  71         seen = set()
  72         first, rest = pattern.split('**/', 1)
  73         if first:
  74             first = iglob(first+'/')
  75         else:
  76             first = ['']
  77         for root in first:
  78             for path in extended_iglob(join_path(root, rest)):
  79                 if path not in seen:
  80                     seen.add(path)
  81                     yield path
  82             for path in extended_iglob(join_path(root, '*', '**/' + rest)):
  83                 if path not in seen:
  84                     seen.add(path)
  85                     yield path
  86     else:
  87         for path in iglob(pattern):
  88             yield path
  89
  90 @cached_function
  91 def file_hash(filename):
  92     path = os.path.normpath(filename.encode("UTF-8"))
  93     m = hashlib.md5(str(len(path)) + ":")
  94     m.update(path)
  95     f = open(filename, 'rb')
  96     try:
  97         data = f.read(65000)
  98         while data:
  99             m.update(data)
 100             data = f.read(65000)
 101     finally:
 102         f.close()
 103     return m.hexdigest()
 104
 105 def parse_list(s):
 106     """
 107     >>> parse_list("a b c")
 108     ['a', 'b', 'c']
 109     >>> parse_list("[a, b, c]")
 110     ['a', 'b', 'c']
 111     >>> parse_list('a " " b')
 112     ['a', ' ', 'b']
 113     >>> parse_list('[a, ",a", "a,", ",", ]')
 114     ['a', ',a', 'a,', ',']
 115     """
 116     if s[0] == '[' and s[-1] == ']':
 117         s = s[1:-1]
 118         delimiter = ','
 119     else:
 120         delimiter = ' '
 121     s, literals = strip_string_literals(s)
 122     def unquote(literal):
 123         literal = literal.strip()
 124         if literal[0] in "'\"":
 125             return literals[literal[1:-1]]
 126         else:
 127             return literal
 128     return [unquote(item) for item in s.split(delimiter) if item.strip()]
 129
 130 transitive_str = object()
 131 transitive_list = object()
 132
 133 distutils_settings = {
 134     'name':                 str,
 135     'sources':              list,
 136     'define_macros':        list,
 137     'undef_macros':         list,
 138     'libraries':            transitive_list,
 139     'library_dirs':         transitive_list,
 140     'runtime_library_dirs': transitive_list,
 141     'include_dirs':         transitive_list,
 142     'extra_objects':        list,
 143     'extra_compile_args':   transitive_list,
 144     'extra_link_args':      transitive_list,
 145     'export_symbols':       list,
 146     'depends':              transitive_list,
 147     'language':             transitive_str,
 148 }
 149
 150 @cython.locals(start=long, end=long)
 151 def line_iter(source):
 152     if isinstance(source, basestring):
 153         start = 0
 154         while True:
 155             end = source.find('\n', start)
 156             if end == -1:
 157                 yield source[start:]
 158                 return
 159             yield source[start:end]
 160             start = end+1
 161     else:
 162         for line in source:
 163             yield line
 164
 165 class DistutilsInfo(object):
 166
 167     def __init__(self, source=None, exn=None):
 168         self.values = {}
 169         if source is not None:
 170             for line in line_iter(source):
 171                 line = line.strip()
 172                 if line != '' and line[0] != '#':
 173                     break
 174                 line = line[1:].strip()
 175                 if line[:10] == 'distutils:':
 176                     line = line[10:]
 177                     ix = line.index('=')
 178                     key = str(line[:ix].strip())
 179                     value = line[ix+1:].strip()
 180                     type = distutils_settings[key]
 181                     if type in (list, transitive_list):
 182                         value = parse_list(value)
 183                         if key == 'define_macros':
 184                             value = [tuple(macro.split('=')) for macro in value]
 185                     self.values[key] = value
 186         elif exn is not None:
 187             for key in distutils_settings:
 188                 if key in ('name', 'sources'):
 189                     continue
 190                 value = getattr(exn, key, None)
 191                 if value:
 192                     self.values[key] = value
 193
 194     def merge(self, other):
 195         if other is None:
 196             return self
 197         for key, value in other.values.items():
 198             type = distutils_settings[key]
 199             if type is transitive_str and key not in self.values:
 200                 self.values[key] = value
 201             elif type is transitive_list:
 202                 if key in self.values:
 203                     all = self.values[key]
 204                     for v in value:
 205                         if v not in all:
 206                             all.append(v)
 207                 else:
 208                     self.values[key] = value
 209         return self
 210
 211     def subs(self, aliases):
 212         if aliases is None:
 213             return self
 214         resolved = DistutilsInfo()
 215         for key, value in self.values.items():
 216             type = distutils_settings[key]
 217             if type in [list, transitive_list]:
 218                 new_value_list = []
 219                 for v in value:
 220                     if v in aliases:
 221                         v = aliases[v]
 222                     if isinstance(v, list):
 223                         new_value_list += v
 224                     else:
 225                         new_value_list.append(v)
 226                 value = new_value_list
 227             else:
 228                 if value in aliases:
 229                     value = aliases[value]
 230             resolved.values[key] = value
 231         return resolved
 232
 233     def apply(self, extension):
 234         for key, value in self.values.items():
 235             type = distutils_settings[key]
 236             if type in [list, transitive_list]:
 237                 getattr(extension, key).extend(value)
 238             else:
 239                 setattr(extension, key, value)
 240
 241 @cython.locals(start=long, q=long, single_q=long, double_q=long, hash_mark=long,
 242                end=long, k=long, counter=long, quote_len=long)
 243 def strip_string_literals(code, prefix='__Pyx_L'):
 244     """
 245     Normalizes every string literal to be of the form '__Pyx_Lxxx',
 246     returning the normalized code and a mapping of labels to
 247     string literals.
 248     """
 249     new_code = []
 250     literals = {}
 251     counter = 0
 252     start = q = 0
 253     in_quote = False
 254     hash_mark = single_q = double_q = -1
 255     code_len = len(code)
 256
 257     while True:
 258         if hash_mark < q:
 259             hash_mark = code.find('#', q)
 260         if single_q < q:
 261             single_q = code.find("'", q)
 262         if double_q < q:
 263             double_q = code.find('"', q)
 264         q = min(single_q, double_q)
 265         if q == -1: q = max(single_q, double_q)
 266
 267         # We're done.
 268         if q == -1 and hash_mark == -1:
 269             new_code.append(code[start:])
 270             break
 271
 272         # Try to close the quote.
 273         elif in_quote:
 274             if code[q-1] == u'\\':
 275                 k = 2
 276                 while q >= k and code[q-k] == u'\\':
 277                     k += 1
 278                 if k % 2 == 0:
 279                     q += 1
 280                     continue
 281             if code[q] == quote_type and (quote_len == 1 or (code_len > q + 2 and quote_type == code[q+1] == code[q+2])):
 282                 counter += 1
 283                 label = "%s%s_" % (prefix, counter)
 284                 literals[label] = code[start+quote_len:q]
 285                 full_quote = code[q:q+quote_len]
 286                 new_code.append(full_quote)
 287                 new_code.append(label)
 288                 new_code.append(full_quote)
 289                 q += quote_len
 290                 in_quote = False
 291                 start = q
 292             else:
 293                 q += 1
 294
 295         # Process comment.
 296         elif -1 != hash_mark and (hash_mark < q or q == -1):
 297             new_code.append(code[start:hash_mark+1])
 298             end = code.find('\n', hash_mark)
 299             counter += 1
 300             label = "%s%s_" % (prefix, counter)
 301             if end == -1:
 302                 end_or_none = None
 303             else:
 304                 end_or_none = end
 305             literals[label] = code[hash_mark+1:end_or_none]
 306             new_code.append(label)
 307             if end == -1:
 308                 break
 309             start = q = end
 310
 311         # Open the quote.
 312         else:
 313             if code_len >= q+3 and (code[q] == code[q+1] == code[q+2]):
 314                 quote_len = 3
 315             else:
 316                 quote_len = 1
 317             in_quote = True
 318             quote_type = code[q]
 319             new_code.append(code[start:q])
 320             start = q
 321             q += quote_len
 322
 323     return "".join(new_code), literals
 324
 325
 326 dependancy_regex = re.compile(r"(?:^from +([0-9a-zA-Z_.]+) +cimport)|"
 327                               r"(?:^cimport +([0-9a-zA-Z_.]+)\b)|"
 328                               r"(?:^cdef +extern +from +['\"]([^'\"]+)['\"])|"
 329                               r"(?:^include +['\"]([^'\"]+)['\"])", re.M)
 330
 331 def normalize_existing(base_path, rel_paths):
 332     return normalize_existing0(os.path.dirname(base_path), tuple(set(rel_paths)))
 333
 334 @cached_function
 335 def normalize_existing0(base_dir, rel_paths):
 336     normalized = []
 337     for rel in rel_paths:
 338         path = join_path(base_dir, rel)
 339         if path_exists(path):
 340             normalized.append(os.path.normpath(path))
 341         else:
 342             normalized.append(rel)
 343     return normalized
 344
 345 def resolve_depends(depends, include_dirs):
 346     include_dirs = tuple(include_dirs)
 347     resolved = []
 348     for depend in depends:
 349         path = resolve_depend(depend, include_dirs)
 350         if path is not None:
 351             resolved.append(path)
 352     return resolved
 353
 354 @cached_function
 355 def resolve_depend(depend, include_dirs):
 356     if depend[0] == '<' and depend[-1] == '>':
 357         return None
 358     for dir in include_dirs:
 359         path = join_path(dir, depend)
 360         if path_exists(path):
 361             return os.path.normpath(path)
 362     return None
 363
 364 @cached_function
 365 def package(filename):
 366     dir = os.path.dirname(os.path.abspath(str(filename)))
 367     if dir != filename and path_exists(join_path(dir, '__init__.py')):
 368         return package(dir) + (os.path.basename(dir),)
 369     else:
 370         return ()
 371
 372 @cached_function
 373 def fully_qualified_name(filename):
 374     module = os.path.splitext(os.path.basename(filename))[0]
 375     return '.'.join(package(filename) + (module,))
 376
 377
 378 @cached_function
 379 def parse_dependencies(source_filename):
 380     # Actual parsing is way to slow, so we use regular expressions.
 381     # The only catch is that we must strip comments and string
 382     # literals ahead of time.
 383     fh = Utils.open_source_file(source_filename, "rU", error_handling='ignore')
 384     try:
 385         source = fh.read()
 386     finally:
 387         fh.close()
 388     distutils_info = DistutilsInfo(source)
 389     source, literals = strip_string_literals(source)
 390     source = source.replace('\\\n', ' ').replace('\t', ' ')
 391
 392     # TODO: pure mode
 393     cimports = []
 394     includes = []
 395     externs  = []
 396     for m in dependancy_regex.finditer(source):
 397         cimport_from, cimport, extern, include = m.groups()
 398         if cimport_from:
 399             cimports.append(cimport_from)
 400         elif cimport:
 401             cimports.append(cimport)
 402         elif extern:
 403             externs.append(literals[extern])
 404         else:
 405             includes.append(literals[include])
 406     return cimports, includes, externs, distutils_info
 407
 408
 409 class DependencyTree(object):
 410
 411     def __init__(self, context, quiet=False):
 412         self.context = context
 413         self.quiet = quiet
 414         self._transitive_cache = {}
 415
 416     def parse_dependencies(self, source_filename):
 417         return parse_dependencies(source_filename)
 418
 419     @cached_method
 420     def included_files(self, filename):
 421         # This is messy because included files are textually included, resolving
 422         # cimports (but not includes) relative to the including file.
 423         all = set()
 424         for include in self.parse_dependencies(filename)[1]:
 425             include_path = join_path(os.path.dirname(filename), include)
 426             if not path_exists(include_path):
 427                 include_path = self.context.find_include_file(include, None)
 428             if include_path:
 429                 if '.' + os.path.sep in include_path:
 430                     include_path = os.path.normpath(include_path)
 431                 all.add(include_path)
 432                 all.update(self.included_files(include_path))
 433             elif not self.quiet:
 434                 print("Unable to locate '%s' referenced from '%s'" % (filename, include))
 435         return all
 436
 437     @cached_method
 438     def cimports_and_externs(self, filename):
 439         # This is really ugly. Nested cimports are resolved with respect to the
 440         # includer, but includes are resolved with respect to the includee.
 441         cimports, includes, externs = self.parse_dependencies(filename)[:3]
 442         cimports = set(cimports)
 443         externs = set(externs)
 444         for include in self.included_files(filename):
 445             included_cimports, included_externs = self.cimports_and_externs(include)
 446             cimports.update(included_cimports)
 447             externs.update(included_externs)
 448         return tuple(cimports), normalize_existing(filename, externs)
 449
 450     def cimports(self, filename):
 451         return self.cimports_and_externs(filename)[0]
 452
 453     def package(self, filename):
 454         return package(filename)
 455
 456     def fully_qualified_name(self, filename):
 457         return fully_qualified_name(filename)
 458
 459     @cached_method
 460     def find_pxd(self, module, filename=None):
 461         is_relative = module[0] == '.'
 462         if is_relative and not filename:
 463             raise NotImplementedError("New relative imports.")
 464         if filename is not None:
 465             module_path = module.split('.')
 466             if is_relative:
 467                 module_path.pop(0)  # just explicitly relative
 468             package_path = list(self.package(filename))
 469             while module_path and not module_path[0]:
 470                 try:
 471                     package_path.pop()
 472                 except IndexError:
 473                     return None   # FIXME: error?
 474                 module_path.pop(0)
 475             relative = '.'.join(package_path + module_path)
 476             pxd = self.context.find_pxd_file(relative, None)
 477             if pxd:
 478                 return pxd
 479         if is_relative:
 480             return None   # FIXME: error?
 481         return self.context.find_pxd_file(module, None)
 482
 483     @cached_method
 484     def cimported_files(self, filename):
 485         if filename[-4:] == '.pyx' and path_exists(filename[:-4] + '.pxd'):
 486             pxd_list = [filename[:-4] + '.pxd']
 487         else:
 488             pxd_list = []
 489         for module in self.cimports(filename):
 490             if module[:7] == 'cython.' or module == 'cython':
 491                 continue
 492             pxd_file = self.find_pxd(module, filename)
 493             if pxd_file is not None:
 494                 pxd_list.append(pxd_file)
 495             elif not self.quiet:
 496                 print("missing cimport in module '%s': %s" % (module, filename))
 497         return tuple(pxd_list)
 498
 499     @cached_method
 500     def immediate_dependencies(self, filename):
 501         all = set([filename])
 502         all.update(self.cimported_files(filename))
 503         all.update(self.included_files(filename))
 504         return all
 505
 506     def all_dependencies(self, filename):
 507         return self.transitive_merge(filename, self.immediate_dependencies, set.union)
 508
 509     @cached_method
 510     def timestamp(self, filename):
 511         return os.path.getmtime(filename)
 512
 513     def extract_timestamp(self, filename):
 514         return self.timestamp(filename), filename
 515
 516     def newest_dependency(self, filename):
 517         return max([self.extract_timestamp(f) for f in self.all_dependencies(filename)])
 518
 519     def transitive_fingerprint(self, filename, extra=None):
 520         try:
 521             m = hashlib.md5(__version__)
 522             m.update(file_hash(filename))
 523             for x in sorted(self.all_dependencies(filename)):
 524                 if os.path.splitext(x)[1] not in ('.c', '.cpp', '.h'):
 525                     m.update(file_hash(x))
 526             if extra is not None:
 527                 m.update(str(extra))
 528             return m.hexdigest()
 529         except IOError:
 530             return None
 531
 532     def distutils_info0(self, filename):
 533         info = self.parse_dependencies(filename)[3]
 534         externs = self.cimports_and_externs(filename)[1]
 535         if externs:
 536             if 'depends' in info.values:
 537                 info.values['depends'] = list(set(info.values['depends']).union(externs))
 538             else:
 539                 info.values['depends'] = list(externs)
 540         return info
 541
 542     def distutils_info(self, filename, aliases=None, base=None):
 543         return (self.transitive_merge(filename, self.distutils_info0, DistutilsInfo.merge)
 544             .subs(aliases)
 545             .merge(base))
 546
 547     def transitive_merge(self, node, extract, merge):
 548         try:
 549             seen = self._transitive_cache[extract, merge]
 550         except KeyError:
 551             seen = self._transitive_cache[extract, merge] = {}
 552         return self.transitive_merge_helper(
 553             node, extract, merge, seen, {}, self.cimported_files)[0]
 554
 555     def transitive_merge_helper(self, node, extract, merge, seen, stack, outgoing):
 556         if node in seen:
 557             return seen[node], None
 558         deps = extract(node)
 559         if node in stack:
 560             return deps, node
 561         try:
 562             stack[node] = len(stack)
 563             loop = None
 564             for next in outgoing(node):
 565                 sub_deps, sub_loop = self.transitive_merge_helper(next, extract, merge, seen, stack, outgoing)
 566                 if sub_loop is not None:
 567                     if loop is not None and stack[loop] < stack[sub_loop]:
 568                         pass
 569                     else:
 570                         loop = sub_loop
 571                 deps = merge(deps, sub_deps)
 572             if loop == node:
 573                 loop = None
 574             if loop is None:
 575                 seen[node] = deps
 576             return deps, loop
 577         finally:
 578             del stack[node]
 579
 580 _dep_tree = None
 581 def create_dependency_tree(ctx=None, quiet=False):
 582     global _dep_tree
 583     if _dep_tree is None:
 584         if ctx is None:
 585             ctx = Context(["."], CompilationOptions(default_options))
 586         _dep_tree = DependencyTree(ctx, quiet=quiet)
 587     return _dep_tree
 588
 589 # This may be useful for advanced users?
 590 def create_extension_list(patterns, exclude=[], ctx=None, aliases=None, quiet=False, exclude_failures=False):
 591     if not isinstance(patterns, (list, tuple)):
 592         patterns = [patterns]
 593     explicit_modules = set([m.name for m in patterns if isinstance(m, Extension)])
 594     seen = set()
 595     deps = create_dependency_tree(ctx, quiet=quiet)
 596     to_exclude = set()
 597     if not isinstance(exclude, list):
 598         exclude = [exclude]
 599     for pattern in exclude:
 600         to_exclude.update(map(os.path.abspath, extended_iglob(pattern)))
 601     module_list = []
 602     for pattern in patterns:
 603         if isinstance(pattern, str):
 604             filepattern = pattern
 605             template = None
 606             name = '*'
 607             base = None
 608             exn_type = Extension
 609         elif isinstance(pattern, Extension):
 610             filepattern = pattern.sources[0]
 611             if os.path.splitext(filepattern)[1] not in ('.py', '.pyx'):
 612                 # ignore non-cython modules
 613                 module_list.append(pattern)
 614                 continue
 615             template = pattern
 616             name = template.name
 617             base = DistutilsInfo(exn=template)
 618             exn_type = template.__class__
 619         else:
 620             raise TypeError(pattern)
 621         for file in extended_iglob(filepattern):
 622             if os.path.abspath(file) in to_exclude:
 623                 continue
 624             pkg = deps.package(file)
 625             if '*' in name:
 626                 module_name = deps.fully_qualified_name(file)
 627                 if module_name in explicit_modules:
 628                     continue
 629             else:
 630                 module_name = name
 631             if module_name not in seen:
 632                 try:
 633                     kwds = deps.distutils_info(file, aliases, base).values
 634                 except Exception:
 635                     if exclude_failures:
 636                         continue
 637                     raise
 638                 if base is not None:
 639                     for key, value in base.values.items():
 640                         if key not in kwds:
 641                             kwds[key] = value
 642                 sources = [file]
 643                 if template is not None:
 644                     sources += template.sources[1:]
 645                 if 'sources' in kwds:
 646                     # allow users to add .c files etc.
 647                     for source in kwds['sources']:
 648                         source = encode_filename_in_py2(source)
 649                         if source not in sources:
 650                             sources.append(source)
 651                     del kwds['sources']
 652                 if 'depends' in kwds:
 653                     depends = resolve_depends(kwds['depends'], (kwds.get('include_dirs') or []) + [find_root_package_dir(file)])
 654                     if template is not None:
 655                         # Always include everything from the template.
 656                         depends = list(set(template.depends).union(set(depends)))
 657                     kwds['depends'] = depends
 658                 module_list.append(exn_type(
 659                         name=module_name,
 660                         sources=sources,
 661                         **kwds))
 662                 m = module_list[-1]
 663                 seen.add(name)
 664     return module_list
 665
 666 # This is the user-exposed entry point.
 667 def cythonize(module_list, exclude=[], nthreads=0, aliases=None, quiet=False, force=False,
 668               exclude_failures=False, **options):
 669     """
 670     Compile a set of source modules into C/C++ files and return a list of distutils
 671     Extension objects for them.
 672
 673     As module list, pass either a glob pattern, a list of glob patterns or a list of
 674     Extension objects.  The latter allows you to configure the extensions separately
 675     through the normal distutils options.
 676
 677     When using glob patterns, you can exclude certain module names explicitly
 678     by passing them into the 'exclude' option.
 679
 680     For parallel compilation, set the 'nthreads' option to the number of
 681     concurrent builds.
 682
 683     For a broad 'try to compile' mode that ignores compilation failures and
 684     simply excludes the failed extensions, pass 'exclude_failures=True'. Note
 685     that this only really makes sense for compiling .py files which can also
 686     be used without compilation.
 687
 688     Additional compilation options can be passed as keyword arguments.
 689     """
 690     if 'include_path' not in options:
 691         options['include_path'] = ['.']
 692     if 'common_utility_include_dir' in options:
 693         if options.get('cache'):
 694             raise NotImplementedError("common_utility_include_dir does not yet work with caching")
 695         if not os.path.exists(options['common_utility_include_dir']):
 696             os.makedirs(options['common_utility_include_dir'])
 697     c_options = CompilationOptions(**options)
 698     cpp_options = CompilationOptions(**options); cpp_options.cplus = True
 699     ctx = c_options.create_context()
 700     options = c_options
 701     module_list = create_extension_list(
 702         module_list,
 703         exclude=exclude,
 704         ctx=ctx,
 705         quiet=quiet,
 706         exclude_failures=exclude_failures,
 707         aliases=aliases)
 708     deps = create_dependency_tree(ctx, quiet=quiet)
 709     build_dir = getattr(options, 'build_dir', None)
 710     modules_by_cfile = {}
 711     to_compile = []
 712     for m in module_list:
 713         if build_dir:
 714             root = os.path.realpath(os.path.abspath(find_root_package_dir(m.sources[0])))
 715             def copy_to_build_dir(filepath, root=root):
 716                 filepath_abs = os.path.realpath(os.path.abspath(filepath))
 717                 if os.path.isabs(filepath):
 718                     filepath = filepath_abs
 719                 if filepath_abs.startswith(root):
 720                     mod_dir = os.path.join(build_dir,
 721                             os.path.dirname(_relpath(filepath, root)))
 722                     if not os.path.isdir(mod_dir):
 723                         os.makedirs(mod_dir)
 724                     shutil.copy(filepath, mod_dir)
 725             for dep in m.depends:
 726                 copy_to_build_dir(dep)
 727
 728         new_sources = []
 729         for source in m.sources:
 730             base, ext = os.path.splitext(source)
 731             if ext in ('.pyx', '.py'):
 732                 if m.language == 'c++':
 733                     c_file = base + '.cpp'
 734                     options = cpp_options
 735                 else:
 736                     c_file = base + '.c'
 737                     options = c_options
 738
 739                 # setup for out of place build directory if enabled
 740                 if build_dir:
 741                     c_file = os.path.join(build_dir, c_file)
 742                     dir = os.path.dirname(c_file)
 743                     if not os.path.isdir(dir):
 744                         os.makedirs(dir)
 745
 746                 if os.path.exists(c_file):
 747                     c_timestamp = os.path.getmtime(c_file)
 748                 else:
 749                     c_timestamp = -1
 750
 751                 # Priority goes first to modified files, second to direct
 752                 # dependents, and finally to indirect dependents.
 753                 if c_timestamp < deps.timestamp(source):
 754                     dep_timestamp, dep = deps.timestamp(source), source
 755                     priority = 0
 756                 else:
 757                     dep_timestamp, dep = deps.newest_dependency(source)
 758                     priority = 2 - (dep in deps.immediate_dependencies(source))
 759                 if force or c_timestamp < dep_timestamp:
 760                     if not quiet:
 761                         if source == dep:
 762                             print("Compiling %s because it changed." % source)
 763                         else:
 764                             print("Compiling %s because it depends on %s." % (source, dep))
 765                     if not force and hasattr(options, 'cache'):
 766                         extra = m.language
 767                         fingerprint = deps.transitive_fingerprint(source, extra)
 768                     else:
 769                         fingerprint = None
 770                     to_compile.append((priority, source, c_file, fingerprint, quiet,
 771                                        options, not exclude_failures))
 772                 new_sources.append(c_file)
 773                 if c_file not in modules_by_cfile:
 774                     modules_by_cfile[c_file] = [m]
 775                 else:
 776                     modules_by_cfile[c_file].append(m)
 777             else:
 778                 new_sources.append(source)
 779                 if build_dir:
 780                     copy_to_build_dir(source)
 781         m.sources = new_sources
 782     if hasattr(options, 'cache'):
 783         if not os.path.exists(options.cache):
 784             os.makedirs(options.cache)
 785     to_compile.sort()
 786     if nthreads:
 787         # Requires multiprocessing (or Python >= 2.6)
 788         try:
 789             import multiprocessing
 790             pool = multiprocessing.Pool(nthreads)
 791         except (ImportError, OSError):
 792             print("multiprocessing required for parallel cythonization")
 793             nthreads = 0
 794         else:
 795             pool.map(cythonize_one_helper, to_compile)
 796     if not nthreads:
 797         for args in to_compile:
 798             cythonize_one(*args[1:])
 799     if exclude_failures:
 800         failed_modules = set()
 801         for c_file, modules in modules_by_cfile.iteritems():
 802             if not os.path.exists(c_file):
 803                 failed_modules.update(modules)
 804             elif os.path.getsize(c_file) < 200:
 805                 f = io_open(c_file, 'r', encoding='iso8859-1')
 806                 try:
 807                     if f.read(len('#error ')) == '#error ':
 808                         # dead compilation result
 809                         failed_modules.update(modules)
 810                 finally:
 811                     f.close()
 812         if failed_modules:
 813             for module in failed_modules:
 814                 module_list.remove(module)
 815             print("Failed compilations: %s" % ', '.join(sorted([
 816                 module.name for module in failed_modules])))
 817     if hasattr(options, 'cache'):
 818         cleanup_cache(options.cache, getattr(options, 'cache_size', 1024 * 1024 * 100))
 819     # cythonize() is often followed by the (non-Python-buffered)
 820     # compiler output, flush now to avoid interleaving output.
 821     sys.stdout.flush()
 822     return module_list
 823
 824
 825 if os.environ.get('XML_RESULTS'):
 826     compile_result_dir = os.environ['XML_RESULTS']
 827     def record_results(func):
 828         def with_record(*args):
 829             t = time.time()
 830             success = True
 831             try:
 832                 try:
 833                     func(*args)
 834                 except:
 835                     success = False
 836             finally:
 837                 t = time.time() - t
 838                 module = fully_qualified_name(args[0])
 839                 name = "cythonize." + module
 840                 failures = 1 - success
 841                 if success:
 842                     failure_item = ""
 843                 else:
 844                     failure_item = "failure"
 845                 output = open(os.path.join(compile_result_dir, name + ".xml"), "w")
 846                 output.write("""
 847                     <?xml version="1.0" ?>
 848                     <testsuite name="%(name)s" errors="0" failures="%(failures)s" tests="1" time="%(t)s">
 849                     <testcase classname="%(name)s" name="cythonize">
 850                     %(failure_item)s
 851                     </testcase>
 852                     </testsuite>
 853                 """.strip() % locals())
 854                 output.close()
 855         return with_record
 856 else:
 857     record_results = lambda x: x
 858
 859 # TODO: Share context? Issue: pyx processing leaks into pxd module
 860 @record_results
 861 def cythonize_one(pyx_file, c_file, fingerprint, quiet, options=None, raise_on_failure=True):
 862     from Cython.Compiler.Main import compile, default_options
 863     from Cython.Compiler.Errors import CompileError, PyrexError
 864
 865     if fingerprint:
 866         if not os.path.exists(options.cache):
 867             try:
 868                 os.mkdir(options.cache)
 869             except:
 870                 if not os.path.exists(options.cache):
 871                     raise
 872         # Cython-generated c files are highly compressible.
 873         # (E.g. a compression ratio of about 10 for Sage).
 874         fingerprint_file = join_path(
 875             options.cache, "%s-%s%s" % (os.path.basename(c_file), fingerprint, gzip_ext))
 876         if os.path.exists(fingerprint_file):
 877             if not quiet:
 878                 print("Found compiled %s in cache" % pyx_file)
 879             os.utime(fingerprint_file, None)
 880             g = gzip_open(fingerprint_file, 'rb')
 881             try:
 882                 f = open(c_file, 'wb')
 883                 try:
 884                     shutil.copyfileobj(g, f)
 885                 finally:
 886                     f.close()
 887             finally:
 888                 g.close()
 889             return
 890     if not quiet:
 891         print("Cythonizing %s" % pyx_file)
 892     if options is None:
 893         options = CompilationOptions(default_options)
 894     options.output_file = c_file
 895
 896     any_failures = 0
 897     try:
 898         result = compile([pyx_file], options)
 899         if result.num_errors > 0:
 900             any_failures = 1
 901     except (EnvironmentError, PyrexError), e:
 902         sys.stderr.write('%s\n' % e)
 903         any_failures = 1
 904         # XXX
 905         import traceback
 906         traceback.print_exc()
 907     except Exception:
 908         if raise_on_failure:
 909             raise
 910         import traceback
 911         traceback.print_exc()
 912         any_failures = 1
 913     if any_failures:
 914         if raise_on_failure:
 915             raise CompileError(None, pyx_file)
 916         elif os.path.exists(c_file):
 917             os.remove(c_file)
 918     elif fingerprint:
 919         f = open(c_file, 'rb')
 920         try:
 921             g = gzip_open(fingerprint_file, 'wb')
 922             try:
 923                 shutil.copyfileobj(f, g)
 924             finally:
 925                 g.close()
 926         finally:
 927             f.close()
 928
 929 def cythonize_one_helper(m):
 930     import traceback
 931     try:
 932         return cythonize_one(*m[1:])
 933     except Exception:
 934         traceback.print_exc()
 935         raise
 936
 937 def cleanup_cache(cache, target_size, ratio=.85):
 938     try:
 939         p = subprocess.Popen(['du', '-s', '-k', os.path.abspath(cache)], stdout=subprocess.PIPE)
 940         res = p.wait()
 941         if res == 0:
 942             total_size = 1024 * int(p.stdout.read().strip().split()[0])
 943             if total_size < target_size:
 944                 return
 945     except (OSError, ValueError):
 946         pass
 947     total_size = 0
 948     all = []
 949     for file in os.listdir(cache):
 950         path = join_path(cache, file)
 951         s = os.stat(path)
 952         total_size += s.st_size
 953         all.append((s.st_atime, s.st_size, path))
 954     if total_size > target_size:
 955         for time, size, file in reversed(sorted(all)):
 956             os.unlink(file)
 957             total_size -= size
 958             if total_size < target_size * ratio:
 959                 break