third_party/cython/src/Cython/Utils.py

   1 #
   2 #   Cython -- Things that don't belong
   3 #            anywhere else in particular
   4 #
   5
   6 import os, sys, re, codecs
   7
   8 modification_time = os.path.getmtime
   9
  10 def cached_function(f):
  11     cache = {}
  12     uncomputed = object()
  13     def wrapper(*args):
  14         res = cache.get(args, uncomputed)
  15         if res is uncomputed:
  16             res = cache[args] = f(*args)
  17         return res
  18     return wrapper
  19
  20 def cached_method(f):
  21     cache_name = '__%s_cache' % f.__name__
  22     def wrapper(self, *args):
  23         cache = getattr(self, cache_name, None)
  24         if cache is None:
  25             cache = {}
  26             setattr(self, cache_name, cache)
  27         if args in cache:
  28             return cache[args]
  29         res = cache[args] = f(self, *args)
  30         return res
  31     return wrapper
  32
  33 def replace_suffix(path, newsuf):
  34     base, _ = os.path.splitext(path)
  35     return base + newsuf
  36
  37 def open_new_file(path):
  38     if os.path.exists(path):
  39         # Make sure to create a new file here so we can
  40         # safely hard link the output files.
  41         os.unlink(path)
  42
  43     # we use the ISO-8859-1 encoding here because we only write pure
  44     # ASCII strings or (e.g. for file names) byte encoded strings as
  45     # Unicode, so we need a direct mapping from the first 256 Unicode
  46     # characters to a byte sequence, which ISO-8859-1 provides
  47     return codecs.open(path, "w", encoding="ISO-8859-1")
  48
  49 def castrate_file(path, st):
  50     #  Remove junk contents from an output file after a
  51     #  failed compilation.
  52     #  Also sets access and modification times back to
  53     #  those specified by st (a stat struct).
  54     try:
  55         f = open_new_file(path)
  56     except EnvironmentError:
  57         pass
  58     else:
  59         f.write(
  60             "#error Do not use this file, it is the result of a failed Cython compilation.\n")
  61         f.close()
  62         if st:
  63             os.utime(path, (st.st_atime, st.st_mtime-1))
  64
  65 def file_newer_than(path, time):
  66     ftime = modification_time(path)
  67     return ftime > time
  68
  69 @cached_function
  70 def search_include_directories(dirs, qualified_name, suffix, pos,
  71                                include=False, sys_path=False):
  72     # Search the list of include directories for the given
  73     # file name. If a source file position is given, first
  74     # searches the directory containing that file. Returns
  75     # None if not found, but does not report an error.
  76     # The 'include' option will disable package dereferencing.
  77     # If 'sys_path' is True, also search sys.path.
  78     if sys_path:
  79         dirs = dirs + tuple(sys.path)
  80     if pos:
  81         file_desc = pos[0]
  82         from Cython.Compiler.Scanning import FileSourceDescriptor
  83         if not isinstance(file_desc, FileSourceDescriptor):
  84             raise RuntimeError("Only file sources for code supported")
  85         if include:
  86             dirs = (os.path.dirname(file_desc.filename),) + dirs
  87         else:
  88             dirs = (find_root_package_dir(file_desc.filename),) + dirs
  89
  90     dotted_filename = qualified_name
  91     if suffix:
  92         dotted_filename += suffix
  93     if not include:
  94         names = qualified_name.split('.')
  95         package_names = tuple(names[:-1])
  96         module_name = names[-1]
  97         module_filename = module_name + suffix
  98         package_filename = "__init__" + suffix
  99
 100     for dir in dirs:
 101         path = os.path.join(dir, dotted_filename)
 102         if path_exists(path):
 103             return path
 104         if not include:
 105             package_dir = check_package_dir(dir, package_names)
 106             if package_dir is not None:
 107                 path = os.path.join(package_dir, module_filename)
 108                 if path_exists(path):
 109                     return path
 110                 path = os.path.join(dir, package_dir, module_name,
 111                                     package_filename)
 112                 if path_exists(path):
 113                     return path
 114     return None
 115
 116
 117 @cached_function
 118 def find_root_package_dir(file_path):
 119     dir = os.path.dirname(file_path)
 120     if file_path == dir:
 121         return dir
 122     elif is_package_dir(dir):
 123         return find_root_package_dir(dir)
 124     else:
 125         return dir
 126
 127 @cached_function
 128 def check_package_dir(dir, package_names):
 129     for dirname in package_names:
 130         dir = os.path.join(dir, dirname)
 131         if not is_package_dir(dir):
 132             return None
 133     return dir
 134
 135 @cached_function
 136 def is_package_dir(dir_path):
 137     for filename in ("__init__.py",
 138                      "__init__.pyx",
 139                      "__init__.pxd"):
 140         path = os.path.join(dir_path, filename)
 141         if path_exists(path):
 142             return 1
 143
 144 @cached_function
 145 def path_exists(path):
 146     # try on the filesystem first
 147     if os.path.exists(path):
 148         return True
 149     # figure out if a PEP 302 loader is around
 150     try:
 151         loader = __loader__
 152         # XXX the code below assumes a 'zipimport.zipimporter' instance
 153         # XXX should be easy to generalize, but too lazy right now to write it
 154         archive_path = getattr(loader, 'archive', None)
 155         if archive_path:
 156             normpath = os.path.normpath(path)
 157             if normpath.startswith(archive_path):
 158                 arcname = normpath[len(archive_path)+1:]
 159                 try:
 160                     loader.get_data(arcname)
 161                     return True
 162                 except IOError:
 163                     return False
 164     except NameError:
 165         pass
 166     return False
 167
 168 # file name encodings
 169
 170 def decode_filename(filename):
 171     if isinstance(filename, unicode):
 172         return filename
 173     try:
 174         filename_encoding = sys.getfilesystemencoding()
 175         if filename_encoding is None:
 176             filename_encoding = sys.getdefaultencoding()
 177         filename = filename.decode(filename_encoding)
 178     except UnicodeDecodeError:
 179         pass
 180     return filename
 181
 182 # support for source file encoding detection
 183
 184 _match_file_encoding = re.compile(u"coding[:=]\s*([-\w.]+)").search
 185
 186 def detect_file_encoding(source_filename):
 187     f = open_source_file(source_filename, encoding="UTF-8", error_handling='ignore')
 188     try:
 189         return detect_opened_file_encoding(f)
 190     finally:
 191         f.close()
 192
 193 def detect_opened_file_encoding(f):
 194     # PEPs 263 and 3120
 195     # Most of the time the first two lines fall in the first 250 chars,
 196     # and this bulk read/split is much faster.
 197     lines = f.read(250).split("\n")
 198     if len(lines) > 2:
 199         m = _match_file_encoding(lines[0]) or _match_file_encoding(lines[1])
 200         if m:
 201             return m.group(1)
 202         else:
 203             return "UTF-8"
 204     else:
 205         # Fallback to one-char-at-a-time detection.
 206         f.seek(0)
 207         chars = []
 208         for i in range(2):
 209             c = f.read(1)
 210             while c and c != u'\n':
 211                 chars.append(c)
 212                 c = f.read(1)
 213             encoding = _match_file_encoding(u''.join(chars))
 214             if encoding:
 215                 return encoding.group(1)
 216     return "UTF-8"
 217
 218
 219 def skip_bom(f):
 220     """
 221     Read past a BOM at the beginning of a source file.
 222     This could be added to the scanner, but it's *substantially* easier
 223     to keep it at this level.
 224     """
 225     if f.read(1) != u'\uFEFF':
 226         f.seek(0)
 227
 228
 229 normalise_newlines = re.compile(u'\r\n?|\n').sub
 230
 231
 232 class NormalisedNewlineStream(object):
 233     """The codecs module doesn't provide universal newline support.
 234     This class is used as a stream wrapper that provides this
 235     functionality.  The new 'io' in Py2.6+/3.x supports this out of the
 236     box.
 237     """
 238
 239     def __init__(self, stream):
 240         # let's assume .read() doesn't change
 241         self.stream = stream
 242         self._read = stream.read
 243         self.close = stream.close
 244         self.encoding = getattr(stream, 'encoding', 'UTF-8')
 245
 246     def read(self, count=-1):
 247         data = self._read(count)
 248         if u'\r' not in data:
 249             return data
 250         if data.endswith(u'\r'):
 251             # may be missing a '\n'
 252             data += self._read(1)
 253         return normalise_newlines(u'\n', data)
 254
 255     def readlines(self):
 256         content = []
 257         data = self.read(0x1000)
 258         while data:
 259             content.append(data)
 260             data = self.read(0x1000)
 261
 262         return u''.join(content).splitlines(True)
 263
 264     def seek(self, pos):
 265         if pos == 0:
 266             self.stream.seek(0)
 267         else:
 268             raise NotImplementedError
 269
 270
 271 io = None
 272 if sys.version_info >= (2,6):
 273     try:
 274         import io
 275     except ImportError:
 276         pass
 277
 278
 279 def open_source_file(source_filename, mode="r",
 280                      encoding=None, error_handling=None,
 281                      require_normalised_newlines=True):
 282     if encoding is None:
 283         # Most of the time the coding is unspecified, so be optimistic that
 284         # it's UTF-8.
 285         f = open_source_file(source_filename, encoding="UTF-8", mode=mode, error_handling='ignore')
 286         encoding = detect_opened_file_encoding(f)
 287         if (encoding == "UTF-8"
 288                 and error_handling == 'ignore'
 289                 and require_normalised_newlines):
 290             f.seek(0)
 291             skip_bom(f)
 292             return f
 293         else:
 294             f.close()
 295     #
 296     if not os.path.exists(source_filename):
 297         try:
 298             loader = __loader__
 299             if source_filename.startswith(loader.archive):
 300                 return open_source_from_loader(
 301                     loader, source_filename,
 302                     encoding, error_handling,
 303                     require_normalised_newlines)
 304         except (NameError, AttributeError):
 305             pass
 306     #
 307     if io is not None:
 308         stream = io.open(source_filename, mode=mode,
 309                          encoding=encoding, errors=error_handling)
 310     else:
 311         # codecs module doesn't have universal newline support
 312         stream = codecs.open(source_filename, mode=mode,
 313                              encoding=encoding, errors=error_handling)
 314         if require_normalised_newlines:
 315             stream = NormalisedNewlineStream(stream)
 316     skip_bom(stream)
 317     return stream
 318
 319
 320 def open_source_from_loader(loader,
 321                             source_filename,
 322                             encoding=None, error_handling=None,
 323                             require_normalised_newlines=True):
 324     nrmpath = os.path.normpath(source_filename)
 325     arcname = nrmpath[len(loader.archive)+1:]
 326     data = loader.get_data(arcname)
 327     if io is not None:
 328         return io.TextIOWrapper(io.BytesIO(data),
 329                                 encoding=encoding,
 330                                 errors=error_handling)
 331     else:
 332         try:
 333             import cStringIO as StringIO
 334         except ImportError:
 335             import StringIO
 336         reader = codecs.getreader(encoding)
 337         stream = reader(StringIO.StringIO(data))
 338         if require_normalised_newlines:
 339             stream = NormalisedNewlineStream(stream)
 340         return stream
 341
 342 def str_to_number(value):
 343     # note: this expects a string as input that was accepted by the
 344     # parser already
 345     if len(value) < 2:
 346         value = int(value, 0)
 347     elif value[0] == '0':
 348         if value[1] in 'xX':
 349             # hex notation ('0x1AF')
 350             value = int(value[2:], 16)
 351         elif value[1] in 'oO':
 352             # Py3 octal notation ('0o136')
 353             value = int(value[2:], 8)
 354         elif value[1] in 'bB':
 355             # Py3 binary notation ('0b101')
 356             value = int(value[2:], 2)
 357         else:
 358             # Py2 octal notation ('0136')
 359             value = int(value, 8)
 360     else:
 361         value = int(value, 0)
 362     return value
 363
 364 def long_literal(value):
 365     if isinstance(value, basestring):
 366         value = str_to_number(value)
 367     return not -2**31 <= value < 2**31
 368
 369 # all() and any() are new in 2.5
 370 try:
 371     # Make sure to bind them on the module, as they will be accessed as
 372     # attributes
 373     all = all
 374     any = any
 375 except NameError:
 376     def all(items):
 377         for item in items:
 378             if not item:
 379                 return False
 380         return True
 381
 382     def any(items):
 383         for item in items:
 384             if item:
 385                 return True
 386         return False
 387
 388 @cached_function
 389 def get_cython_cache_dir():
 390     """get the cython cache dir
 391
 392     Priority:
 393
 394     1. CYTHON_CACHE_DIR
 395     2. (OS X): ~/Library/Caches/Cython
 396        (posix not OS X): XDG_CACHE_HOME/cython if XDG_CACHE_HOME defined
 397     3. ~/.cython
 398
 399     """
 400     if 'CYTHON_CACHE_DIR' in os.environ:
 401         return os.environ['CYTHON_CACHE_DIR']
 402
 403     parent = None
 404     if os.name == 'posix':
 405         if sys.platform == 'darwin':
 406             parent = os.path.expanduser('~/Library/Caches')
 407         else:
 408             # this could fallback on ~/.cache
 409             parent = os.environ.get('XDG_CACHE_HOME')
 410
 411     if parent and os.path.isdir(parent):
 412         return os.path.join(parent, 'cython')
 413
 414     # last fallback: ~/.cython
 415     return os.path.expanduser(os.path.join('~', '.cython'))