third_party/cython/src/Cython/Compiler/Code.py

   1 # cython: language_level = 2
   2 #
   3 #   Code output module
   4 #
   5
   6 import cython
   7 cython.declare(os=object, re=object, operator=object,
   8                Naming=object, Options=object, StringEncoding=object,
   9                Utils=object, SourceDescriptor=object, StringIOTree=object,
  10                DebugFlags=object, basestring=object)
  11
  12 import os
  13 import re
  14 import sys
  15 from string import Template
  16 import operator
  17 import textwrap
  18
  19 try:
  20     import hashlib
  21 except ImportError:
  22     import md5 as hashlib
  23
  24 import Naming
  25 import Options
  26 import StringEncoding
  27 from Cython import Utils
  28 from Scanning import SourceDescriptor
  29 from Cython.StringIOTree import StringIOTree
  30 import DebugFlags
  31
  32 try:
  33     from __builtin__ import basestring
  34 except ImportError:
  35     from builtins import str as basestring
  36
  37 KEYWORDS_MUST_BE_BYTES = sys.version_info < (2,7)
  38
  39
  40 non_portable_builtins_map = {
  41     # builtins that have different names in different Python versions
  42     'bytes'         : ('PY_MAJOR_VERSION < 3',  'str'),
  43     'unicode'       : ('PY_MAJOR_VERSION >= 3', 'str'),
  44     'basestring'    : ('PY_MAJOR_VERSION >= 3', 'str'),
  45     'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
  46     'raw_input'     : ('PY_MAJOR_VERSION >= 3', 'input'),
  47     'BaseException' : ('PY_VERSION_HEX < 0x02050000', 'Exception'),
  48     }
  49
  50 basicsize_builtins_map = {
  51     # builtins whose type has a different tp_basicsize than sizeof(...)
  52     'PyTypeObject' : 'PyHeapTypeObject',
  53     }
  54
  55 uncachable_builtins = [
  56     # builtin names that cannot be cached because they may or may not
  57     # be available at import time
  58     'WindowsError',
  59     ]
  60
  61 modifier_output_mapper = {
  62     'inline': 'CYTHON_INLINE'
  63 }.get
  64
  65 is_self_assignment = re.compile(r" *(\w+) = (\1);\s*$").match
  66
  67
  68 def get_utility_dir():
  69     # make this a function and not global variables:
  70     # http://trac.cython.org/cython_trac/ticket/475
  71     Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
  72     return os.path.join(Cython_dir, "Utility")
  73
  74
  75 class UtilityCodeBase(object):
  76     """
  77     Support for loading utility code from a file.
  78
  79     Code sections in the file can be specified as follows:
  80
  81         ##### MyUtility.proto #####
  82
  83         [proto declarations]
  84
  85         ##### MyUtility.init #####
  86
  87         [code run at module initialization]
  88
  89         ##### MyUtility #####
  90         #@requires: MyOtherUtility
  91         #@substitute: naming
  92
  93         [definitions]
  94
  95     for prototypes and implementation respectively.  For non-python or
  96     -cython files backslashes should be used instead.  5 to 30 comment
  97     characters may be used on either side.
  98
  99     If the @cname decorator is not used and this is a CythonUtilityCode,
 100     one should pass in the 'name' keyword argument to be used for name
 101     mangling of such entries.
 102     """
 103
 104     is_cython_utility = False
 105     requires = None
 106     _utility_cache = {}
 107
 108     @classmethod
 109     def _add_utility(cls, utility, type, lines, begin_lineno, tags=None):
 110         if utility is None:
 111             return
 112
 113         code = '\n'.join(lines)
 114         if tags and 'substitute' in tags and tags['substitute'] == set(['naming']):
 115             del tags['substitute']
 116             try:
 117                 code = Template(code).substitute(vars(Naming))
 118             except (KeyError, ValueError), e:
 119                 raise RuntimeError("Error parsing templated utility code of type '%s' at line %d: %s" % (
 120                     type, begin_lineno, e))
 121
 122         # remember correct line numbers at least until after templating
 123         code = '\n' * begin_lineno + code
 124
 125         if type == 'proto':
 126             utility[0] = code
 127         elif type == 'impl':
 128             utility[1] = code
 129         else:
 130             all_tags = utility[2]
 131             if KEYWORDS_MUST_BE_BYTES:
 132                 type = type.encode('ASCII')
 133             all_tags[type] = code
 134
 135         if tags:
 136             all_tags = utility[2]
 137             for name, values in tags.items():
 138                 if KEYWORDS_MUST_BE_BYTES:
 139                     name = name.encode('ASCII')
 140                 all_tags.setdefault(name, set()).update(values)
 141
 142     @classmethod
 143     def load_utilities_from_file(cls, path):
 144         utilities = cls._utility_cache.get(path)
 145         if utilities:
 146             return utilities
 147
 148         filename = os.path.join(get_utility_dir(), path)
 149         _, ext = os.path.splitext(path)
 150         if ext in ('.pyx', '.py', '.pxd', '.pxi'):
 151             comment = '#'
 152             replace_comments = re.compile(r'^\s*#.*').sub
 153         else:
 154             comment = '/'
 155             replace_comments = re.compile(r'^\s*//.*|^\s*/\*[^*]*\*/').sub
 156         match_special = re.compile(
 157             (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|'
 158              r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)'
 159                 ) % {'C':comment}).match
 160         match_type = re.compile('(.+)[.](proto|impl|init|cleanup)$').match
 161
 162         f = Utils.open_source_file(filename, encoding='UTF-8')
 163         try:
 164             all_lines = f.readlines()
 165         finally:
 166             f.close()
 167
 168         utilities = {}
 169         lines = []
 170         tags = {}
 171         utility = type = None
 172         begin_lineno = 0
 173
 174         for lineno, line in enumerate(all_lines):
 175             m = match_special(line)
 176             if m:
 177                 if m.group('name'):
 178                     cls._add_utility(utility, type, lines, begin_lineno, tags)
 179
 180                     begin_lineno = lineno + 1
 181                     del lines[:]
 182                     tags.clear()
 183
 184                     name = m.group('name')
 185                     mtype = match_type(name)
 186                     if mtype:
 187                         name, type = mtype.groups()
 188                     else:
 189                         type = 'impl'
 190                     utility = utilities.setdefault(name, [None, None, {}])
 191                 else:
 192                     tags.setdefault(m.group('tag'), set()).add(m.group('value'))
 193                     lines.append('') # keep line number correct
 194             else:
 195                 lines.append(replace_comments('', line).rstrip())
 196
 197         if utility is None:
 198             raise ValueError("Empty utility code file")
 199
 200         # Don't forget to add the last utility code
 201         cls._add_utility(utility, type, lines, begin_lineno, tags)
 202
 203         cls._utility_cache[path] = utilities
 204         return utilities
 205
 206     @classmethod
 207     def load(cls, util_code_name, from_file=None, **kwargs):
 208         """
 209         Load utility code from a file specified by from_file (relative to
 210         Cython/Utility) and name util_code_name.  If from_file is not given,
 211         load it from the file util_code_name.*.  There should be only one
 212         file matched by this pattern.
 213         """
 214         if '::' in util_code_name:
 215             from_file, util_code_name = util_code_name.rsplit('::', 1)
 216         if not from_file:
 217             utility_dir = get_utility_dir()
 218             prefix = util_code_name + '.'
 219             try:
 220                 listing = os.listdir(utility_dir)
 221             except OSError:
 222                 # XXX the code below assumes as 'zipimport.zipimporter' instance
 223                 # XXX should be easy to generalize, but too lazy right now to write it
 224                 import zipfile
 225                 global __loader__
 226                 loader = __loader__
 227                 archive = loader.archive
 228                 fileobj = zipfile.ZipFile(archive)
 229                 listing = [ os.path.basename(name)
 230                             for name in fileobj.namelist()
 231                             if os.path.join(archive, name).startswith(utility_dir)]
 232                 fileobj.close()
 233             files = [ os.path.join(utility_dir, filename)
 234                       for filename in listing
 235                       if filename.startswith(prefix) ]
 236             if not files:
 237                 raise ValueError("No match found for utility code " + util_code_name)
 238             if len(files) > 1:
 239                 raise ValueError("More than one filename match found for utility code " + util_code_name)
 240             from_file = files[0]
 241
 242         utilities = cls.load_utilities_from_file(from_file)
 243         proto, impl, tags = utilities[util_code_name]
 244
 245         if tags:
 246             orig_kwargs = kwargs.copy()
 247             for name, values in tags.items():
 248                 if name in kwargs:
 249                     continue
 250                 # only pass lists when we have to: most argument expect one value or None
 251                 if name == 'requires':
 252                     if orig_kwargs:
 253                         values = [cls.load(dep, from_file, **orig_kwargs)
 254                                   for dep in sorted(values)]
 255                     else:
 256                         # dependencies are rarely unique, so use load_cached() when we can
 257                         values = [cls.load_cached(dep, from_file)
 258                                   for dep in sorted(values)]
 259                 elif not values:
 260                     values = None
 261                 elif len(values) == 1:
 262                     values = values[0]
 263                 kwargs[name] = values
 264
 265         if proto is not None:
 266             kwargs['proto'] = proto
 267         if impl is not None:
 268             kwargs['impl'] = impl
 269
 270         if 'name' not in kwargs:
 271             kwargs['name'] = util_code_name
 272
 273         if 'file' not in kwargs and from_file:
 274             kwargs['file'] = from_file
 275         return cls(**kwargs)
 276
 277     @classmethod
 278     def load_cached(cls, utility_code_name, from_file=None, __cache={}):
 279         """
 280         Calls .load(), but using a per-type cache based on utility name and file name.
 281         """
 282         key = (cls, from_file, utility_code_name)
 283         try:
 284             return __cache[key]
 285         except KeyError:
 286             pass
 287         code = __cache[key] = cls.load(utility_code_name, from_file)
 288         return code
 289
 290     @classmethod
 291     def load_as_string(cls, util_code_name, from_file=None, **kwargs):
 292         """
 293         Load a utility code as a string. Returns (proto, implementation)
 294         """
 295         util = cls.load(util_code_name, from_file, **kwargs)
 296         proto, impl = util.proto, util.impl
 297         return util.format_code(proto), util.format_code(impl)
 298
 299     def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').sub):
 300         """
 301         Format a code section for output.
 302         """
 303         if code_string:
 304             code_string = replace_empty_lines('\n', code_string.strip()) + '\n\n'
 305         return code_string
 306
 307     def __str__(self):
 308         return "<%s(%s)" % (type(self).__name__, self.name)
 309
 310     def get_tree(self):
 311         pass
 312
 313
 314 class UtilityCode(UtilityCodeBase):
 315     """
 316     Stores utility code to add during code generation.
 317
 318     See GlobalState.put_utility_code.
 319
 320     hashes/equals by instance
 321
 322     proto           C prototypes
 323     impl            implemenation code
 324     init            code to call on module initialization
 325     requires        utility code dependencies
 326     proto_block     the place in the resulting file where the prototype should
 327                     end up
 328     name            name of the utility code (or None)
 329     file            filename of the utility code file this utility was loaded
 330                     from (or None)
 331     """
 332
 333     def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
 334                  proto_block='utility_code_proto', name=None, file=None):
 335         # proto_block: Which code block to dump prototype in. See GlobalState.
 336         self.proto = proto
 337         self.impl = impl
 338         self.init = init
 339         self.cleanup = cleanup
 340         self.requires = requires
 341         self._cache = {}
 342         self.specialize_list = []
 343         self.proto_block = proto_block
 344         self.name = name
 345         self.file = file
 346
 347     def __hash__(self):
 348         return hash((self.proto, self.impl))
 349
 350     def __eq__(self, other):
 351         if self is other:
 352             return True
 353         if not isinstance(other, type(self)):
 354             return False
 355
 356         self_proto = getattr(self, 'proto', None)
 357         other_proto = getattr(other, 'proto', None)
 358         return (self_proto, self.impl) == (other_proto, other.impl)
 359
 360     def none_or_sub(self, s, context):
 361         """
 362         Format a string in this utility code with context. If None, do nothing.
 363         """
 364         if s is None:
 365             return None
 366         return s % context
 367
 368     def specialize(self, pyrex_type=None, **data):
 369         # Dicts aren't hashable...
 370         if pyrex_type is not None:
 371             data['type'] = pyrex_type.declaration_code('')
 372             data['type_name'] = pyrex_type.specialization_name()
 373         key = tuple(sorted(data.items()))
 374         try:
 375             return self._cache[key]
 376         except KeyError:
 377             if self.requires is None:
 378                 requires = None
 379             else:
 380                 requires = [r.specialize(data) for r in self.requires]
 381
 382             s = self._cache[key] = UtilityCode(
 383                     self.none_or_sub(self.proto, data),
 384                     self.none_or_sub(self.impl, data),
 385                     self.none_or_sub(self.init, data),
 386                     self.none_or_sub(self.cleanup, data),
 387                     requires,
 388                     self.proto_block)
 389
 390             self.specialize_list.append(s)
 391             return s
 392
 393     def inject_string_constants(self, impl, output):
 394         """Replace 'PYIDENT("xyz")' by a constant Python identifier cname.
 395         """
 396         replacements = {}
 397         def externalise(matchobj):
 398             name = matchobj.group(1)
 399             try:
 400                 cname = replacements[name]
 401             except KeyError:
 402                 cname = replacements[name] = output.get_interned_identifier(
 403                     StringEncoding.EncodedString(name)).cname
 404             return cname
 405
 406         impl = re.sub('PYIDENT\("([^"]+)"\)', externalise, impl)
 407         return bool(replacements), impl
 408
 409     def put_code(self, output):
 410         if self.requires:
 411             for dependency in self.requires:
 412                 output.use_utility_code(dependency)
 413         if self.proto:
 414             output[self.proto_block].put_or_include(
 415                 self.format_code(self.proto),
 416                 '%s_proto' % self.name)
 417         if self.impl:
 418             impl = self.format_code(self.impl)
 419             is_specialised, impl = self.inject_string_constants(impl, output)
 420             if not is_specialised:
 421                 # no module specific adaptations => can be reused
 422                 output['utility_code_def'].put_or_include(
 423                     impl, '%s_impl' % self.name)
 424             else:
 425                 output['utility_code_def'].put(impl)
 426         if self.init:
 427             writer = output['init_globals']
 428             writer.putln("/* %s.init */" % self.name)
 429             if isinstance(self.init, basestring):
 430                 writer.put(self.format_code(self.init))
 431             else:
 432                 self.init(writer, output.module_pos)
 433             writer.putln(writer.error_goto_if_PyErr(output.module_pos))
 434             writer.putln()
 435         if self.cleanup and Options.generate_cleanup_code:
 436             writer = output['cleanup_globals']
 437             if isinstance(self.cleanup, basestring):
 438                 writer.put_or_include(
 439                     self.format_code(self.cleanup),
 440                     '%s_cleanup' % self.name)
 441             else:
 442                 self.cleanup(writer, output.module_pos)
 443
 444
 445 def sub_tempita(s, context, file=None, name=None):
 446     "Run tempita on string s with given context."
 447     if not s:
 448         return None
 449
 450     if file:
 451         context['__name'] = "%s:%s" % (file, name)
 452     elif name:
 453         context['__name'] = name
 454
 455     from Cython.Tempita import sub
 456     return sub(s, **context)
 457
 458 class TempitaUtilityCode(UtilityCode):
 459     def __init__(self, name=None, proto=None, impl=None, init=None, file=None, context=None, **kwargs):
 460         if context is None:
 461             context = {}
 462         proto = sub_tempita(proto, context, file, name)
 463         impl = sub_tempita(impl, context, file, name)
 464         init = sub_tempita(init, context, file, name)
 465         super(TempitaUtilityCode, self).__init__(
 466             proto, impl, init=init, name=name, file=file, **kwargs)
 467
 468     def none_or_sub(self, s, context):
 469         """
 470         Format a string in this utility code with context. If None, do nothing.
 471         """
 472         if s is None:
 473             return None
 474         return sub_tempita(s, context, self.file, self.name)
 475
 476
 477 class LazyUtilityCode(UtilityCodeBase):
 478     """
 479     Utility code that calls a callback with the root code writer when
 480     available. Useful when you only have 'env' but not 'code'.
 481     """
 482
 483     def __init__(self, callback):
 484         self.callback = callback
 485
 486     def put_code(self, globalstate):
 487         utility = self.callback(globalstate.rootwriter)
 488         globalstate.use_utility_code(utility)
 489
 490
 491 class FunctionState(object):
 492     # return_label     string          function return point label
 493     # error_label      string          error catch point label
 494     # continue_label   string          loop continue point label
 495     # break_label      string          loop break point label
 496     # return_from_error_cleanup_label string
 497     # label_counter    integer         counter for naming labels
 498     # in_try_finally   boolean         inside try of try...finally
 499     # exc_vars         (string * 3)    exception variables for reraise, or None
 500     # can_trace        boolean         line tracing is supported in the current context
 501
 502     # Not used for now, perhaps later
 503     def __init__(self, owner, names_taken=set()):
 504         self.names_taken = names_taken
 505         self.owner = owner
 506
 507         self.error_label = None
 508         self.label_counter = 0
 509         self.labels_used = set()
 510         self.return_label = self.new_label()
 511         self.new_error_label()
 512         self.continue_label = None
 513         self.break_label = None
 514         self.yield_labels = []
 515
 516         self.in_try_finally = 0
 517         self.exc_vars = None
 518         self.can_trace = False
 519
 520         self.temps_allocated = [] # of (name, type, manage_ref, static)
 521         self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
 522         self.temps_used_type = {} # name -> (type, manage_ref)
 523         self.temp_counter = 0
 524         self.closure_temps = None
 525
 526         # This is used to collect temporaries, useful to find out which temps
 527         # need to be privatized in parallel sections
 528         self.collect_temps_stack = []
 529
 530         # This is used for the error indicator, which needs to be local to the
 531         # function. It used to be global, which relies on the GIL being held.
 532         # However, exceptions may need to be propagated through 'nogil'
 533         # sections, in which case we introduce a race condition.
 534         self.should_declare_error_indicator = False
 535         self.uses_error_indicator = False
 536
 537     # labels
 538
 539     def new_label(self, name=None):
 540         n = self.label_counter
 541         self.label_counter = n + 1
 542         label = "%s%d" % (Naming.label_prefix, n)
 543         if name is not None:
 544             label += '_' + name
 545         return label
 546
 547     def new_yield_label(self):
 548         label = self.new_label('resume_from_yield')
 549         num_and_label = (len(self.yield_labels) + 1, label)
 550         self.yield_labels.append(num_and_label)
 551         return num_and_label
 552
 553     def new_error_label(self):
 554         old_err_lbl = self.error_label
 555         self.error_label = self.new_label('error')
 556         return old_err_lbl
 557
 558     def get_loop_labels(self):
 559         return (
 560             self.continue_label,
 561             self.break_label)
 562
 563     def set_loop_labels(self, labels):
 564         (self.continue_label,
 565          self.break_label) = labels
 566
 567     def new_loop_labels(self):
 568         old_labels = self.get_loop_labels()
 569         self.set_loop_labels(
 570             (self.new_label("continue"),
 571              self.new_label("break")))
 572         return old_labels
 573
 574     def get_all_labels(self):
 575         return (
 576             self.continue_label,
 577             self.break_label,
 578             self.return_label,
 579             self.error_label)
 580
 581     def set_all_labels(self, labels):
 582         (self.continue_label,
 583          self.break_label,
 584          self.return_label,
 585          self.error_label) = labels
 586
 587     def all_new_labels(self):
 588         old_labels = self.get_all_labels()
 589         new_labels = []
 590         for old_label, name in zip(old_labels, ['continue', 'break', 'return', 'error']):
 591             if old_label:
 592                 new_labels.append(self.new_label(name))
 593             else:
 594                 new_labels.append(old_label)
 595         self.set_all_labels(new_labels)
 596         return old_labels
 597
 598     def use_label(self, lbl):
 599         self.labels_used.add(lbl)
 600
 601     def label_used(self, lbl):
 602         return lbl in self.labels_used
 603
 604     # temp handling
 605
 606     def allocate_temp(self, type, manage_ref, static=False):
 607         """
 608         Allocates a temporary (which may create a new one or get a previously
 609         allocated and released one of the same type). Type is simply registered
 610         and handed back, but will usually be a PyrexType.
 611
 612         If type.is_pyobject, manage_ref comes into play. If manage_ref is set to
 613         True, the temp will be decref-ed on return statements and in exception
 614         handling clauses. Otherwise the caller has to deal with any reference
 615         counting of the variable.
 616
 617         If not type.is_pyobject, then manage_ref will be ignored, but it
 618         still has to be passed. It is recommended to pass False by convention
 619         if it is known that type will never be a Python object.
 620
 621         static=True marks the temporary declaration with "static".
 622         This is only used when allocating backing store for a module-level
 623         C array literals.
 624
 625         A C string referring to the variable is returned.
 626         """
 627         if type.is_const:
 628             type = type.const_base_type
 629         if not type.is_pyobject and not type.is_memoryviewslice:
 630             # Make manage_ref canonical, so that manage_ref will always mean
 631             # a decref is needed.
 632             manage_ref = False
 633
 634         freelist = self.temps_free.get((type, manage_ref))
 635         if freelist is not None and len(freelist) > 0:
 636             result = freelist.pop()
 637         else:
 638             while True:
 639                 self.temp_counter += 1
 640                 result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter)
 641                 if not result in self.names_taken: break
 642             self.temps_allocated.append((result, type, manage_ref, static))
 643         self.temps_used_type[result] = (type, manage_ref)
 644         if DebugFlags.debug_temp_code_comments:
 645             self.owner.putln("/* %s allocated */" % result)
 646
 647         if self.collect_temps_stack:
 648             self.collect_temps_stack[-1].add((result, type))
 649
 650         return result
 651
 652     def release_temp(self, name):
 653         """
 654         Releases a temporary so that it can be reused by other code needing
 655         a temp of the same type.
 656         """
 657         type, manage_ref = self.temps_used_type[name]
 658         freelist = self.temps_free.get((type, manage_ref))
 659         if freelist is None:
 660             freelist = []
 661             self.temps_free[(type, manage_ref)] = freelist
 662         if name in freelist:
 663             raise RuntimeError("Temp %s freed twice!" % name)
 664         freelist.append(name)
 665         if DebugFlags.debug_temp_code_comments:
 666             self.owner.putln("/* %s released */" % name)
 667
 668     def temps_in_use(self):
 669         """Return a list of (cname,type,manage_ref) tuples of temp names and their type
 670         that are currently in use.
 671         """
 672         used = []
 673         for name, type, manage_ref, static in self.temps_allocated:
 674             freelist = self.temps_free.get((type, manage_ref))
 675             if freelist is None or name not in freelist:
 676                 used.append((name, type, manage_ref and type.is_pyobject))
 677         return used
 678
 679     def temps_holding_reference(self):
 680         """Return a list of (cname,type) tuples of temp names and their type
 681         that are currently in use. This includes only temps of a
 682         Python object type which owns its reference.
 683         """
 684         return [(name, type)
 685                 for name, type, manage_ref in self.temps_in_use()
 686                 if manage_ref  and type.is_pyobject]
 687
 688     def all_managed_temps(self):
 689         """Return a list of (cname, type) tuples of refcount-managed Python objects.
 690         """
 691         return [(cname, type)
 692                     for cname, type, manage_ref, static in self.temps_allocated
 693                         if manage_ref]
 694
 695     def all_free_managed_temps(self):
 696         """Return a list of (cname, type) tuples of refcount-managed Python
 697         objects that are not currently in use.  This is used by
 698         try-except and try-finally blocks to clean up temps in the
 699         error case.
 700         """
 701         return [(cname, type)
 702                     for (type, manage_ref), freelist in self.temps_free.items()
 703                         if manage_ref
 704                             for cname in freelist]
 705
 706     def start_collecting_temps(self):
 707         """
 708         Useful to find out which temps were used in a code block
 709         """
 710         self.collect_temps_stack.append(set())
 711
 712     def stop_collecting_temps(self):
 713         return self.collect_temps_stack.pop()
 714
 715     def init_closure_temps(self, scope):
 716         self.closure_temps = ClosureTempAllocator(scope)
 717
 718
 719 class NumConst(object):
 720     """Global info about a Python number constant held by GlobalState.
 721
 722     cname       string
 723     value       string
 724     py_type     string     int, long, float
 725     value_code  string     evaluation code if different from value
 726     """
 727
 728     def __init__(self, cname, value, py_type, value_code=None):
 729         self.cname = cname
 730         self.value = value
 731         self.py_type = py_type
 732         self.value_code = value_code or value
 733
 734
 735 class PyObjectConst(object):
 736     """Global info about a generic constant held by GlobalState.
 737     """
 738     # cname       string
 739     # type        PyrexType
 740
 741     def __init__(self, cname, type):
 742         self.cname = cname
 743         self.type = type
 744
 745
 746 cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=object,
 747                replace_identifier=object, find_alphanums=object)
 748 possible_unicode_identifier = re.compile(ur"(?![0-9])\w+$", re.U).match
 749 possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match
 750 replace_identifier = re.compile(r'[^a-zA-Z0-9_]+').sub
 751 find_alphanums = re.compile('([a-zA-Z0-9]+)').findall
 752
 753 class StringConst(object):
 754     """Global info about a C string constant held by GlobalState.
 755     """
 756     # cname            string
 757     # text             EncodedString or BytesLiteral
 758     # py_strings       {(identifier, encoding) : PyStringConst}
 759
 760     def __init__(self, cname, text, byte_string):
 761         self.cname = cname
 762         self.text = text
 763         self.escaped_value = StringEncoding.escape_byte_string(byte_string)
 764         self.py_strings = None
 765         self.py_versions = []
 766
 767     def add_py_version(self, version):
 768         if not version:
 769             self.py_versions = [2,3]
 770         elif version not in self.py_versions:
 771             self.py_versions.append(version)
 772
 773     def get_py_string_const(self, encoding, identifier=None,
 774                             is_str=False, py3str_cstring=None):
 775         py_strings = self.py_strings
 776         text = self.text
 777
 778         is_str = bool(identifier or is_str)
 779         is_unicode = encoding is None and not is_str
 780
 781         if encoding is None:
 782             # unicode string
 783             encoding_key = None
 784         else:
 785             # bytes or str
 786             encoding = encoding.lower()
 787             if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'):
 788                 encoding = None
 789                 encoding_key = None
 790             else:
 791                 encoding_key = ''.join(find_alphanums(encoding))
 792
 793         key = (is_str, is_unicode, encoding_key, py3str_cstring)
 794         if py_strings is not None:
 795             try:
 796                 return py_strings[key]
 797             except KeyError:
 798                 pass
 799         else:
 800             self.py_strings = {}
 801
 802         if identifier:
 803             intern = True
 804         elif identifier is None:
 805             if isinstance(text, unicode):
 806                 intern = bool(possible_unicode_identifier(text))
 807             else:
 808                 intern = bool(possible_bytes_identifier(text))
 809         else:
 810             intern = False
 811         if intern:
 812             prefix = Naming.interned_prefixes['str']
 813         else:
 814             prefix = Naming.py_const_prefix
 815
 816         if encoding_key:
 817             encoding_prefix = '_%s' % encoding_key
 818         else:
 819             encoding_prefix = ''
 820
 821         pystring_cname = "%s%s%s_%s" % (
 822             prefix,
 823             (is_str and 's') or (is_unicode and 'u') or 'b',
 824             encoding_prefix,
 825             self.cname[len(Naming.const_prefix):])
 826
 827         py_string = PyStringConst(
 828             pystring_cname, encoding, is_unicode, is_str, py3str_cstring, intern)
 829         self.py_strings[key] = py_string
 830         return py_string
 831
 832 class PyStringConst(object):
 833     """Global info about a Python string constant held by GlobalState.
 834     """
 835     # cname       string
 836     # py3str_cstring string
 837     # encoding    string
 838     # intern      boolean
 839     # is_unicode  boolean
 840     # is_str      boolean
 841
 842     def __init__(self, cname, encoding, is_unicode, is_str=False,
 843                  py3str_cstring=None, intern=False):
 844         self.cname = cname
 845         self.py3str_cstring = py3str_cstring
 846         self.encoding = encoding
 847         self.is_str = is_str
 848         self.is_unicode = is_unicode
 849         self.intern = intern
 850
 851     def __lt__(self, other):
 852         return self.cname < other.cname
 853
 854
 855 class GlobalState(object):
 856     # filename_table   {string : int}  for finding filename table indexes
 857     # filename_list    [string]        filenames in filename table order
 858     # input_file_contents dict         contents (=list of lines) of any file that was used as input
 859     #                                  to create this output C code.  This is
 860     #                                  used to annotate the comments.
 861     #
 862     # utility_codes   set                IDs of used utility code (to avoid reinsertion)
 863     #
 864     # declared_cnames  {string:Entry}  used in a transition phase to merge pxd-declared
 865     #                                  constants etc. into the pyx-declared ones (i.e,
 866     #                                  check if constants are already added).
 867     #                                  In time, hopefully the literals etc. will be
 868     #                                  supplied directly instead.
 869     #
 870     # const_cnames_used  dict          global counter for unique constant identifiers
 871     #
 872
 873     # parts            {string:CCodeWriter}
 874
 875
 876     # interned_strings
 877     # consts
 878     # interned_nums
 879
 880     # directives       set             Temporary variable used to track
 881     #                                  the current set of directives in the code generation
 882     #                                  process.
 883
 884     directives = {}
 885
 886     code_layout = [
 887         'h_code',
 888         'filename_table',
 889         'utility_code_proto_before_types',
 890         'numeric_typedefs',          # Let these detailed individual parts stay!,
 891         'complex_type_declarations', # as the proper solution is to make a full DAG...
 892         'type_declarations',         # More coarse-grained blocks would simply hide
 893         'utility_code_proto',        # the ugliness, not fix it
 894         'module_declarations',
 895         'typeinfo',
 896         'before_global_var',
 897         'global_var',
 898         'decls',
 899         'all_the_rest',
 900         'pystring_table',
 901         'cached_builtins',
 902         'cached_constants',
 903         'init_globals',
 904         'init_module',
 905         'cleanup_globals',
 906         'cleanup_module',
 907         'main_method',
 908         'utility_code_def',
 909         'end'
 910     ]
 911
 912
 913     def __init__(self, writer, module_node, emit_linenums=False, common_utility_include_dir=None):
 914         self.filename_table = {}
 915         self.filename_list = []
 916         self.input_file_contents = {}
 917         self.utility_codes = set()
 918         self.declared_cnames = {}
 919         self.in_utility_code_generation = False
 920         self.emit_linenums = emit_linenums
 921         self.common_utility_include_dir = common_utility_include_dir
 922         self.parts = {}
 923         self.module_node = module_node # because some utility code generation needs it
 924                                        # (generating backwards-compatible Get/ReleaseBuffer
 925
 926         self.const_cnames_used = {}
 927         self.string_const_index = {}
 928         self.pyunicode_ptr_const_index = {}
 929         self.num_const_index = {}
 930         self.py_constants = []
 931
 932         assert writer.globalstate is None
 933         writer.globalstate = self
 934         self.rootwriter = writer
 935
 936     def initialize_main_c_code(self):
 937         rootwriter = self.rootwriter
 938         for part in self.code_layout:
 939             self.parts[part] = rootwriter.insertion_point()
 940
 941         if not Options.cache_builtins:
 942             del self.parts['cached_builtins']
 943         else:
 944             w = self.parts['cached_builtins']
 945             w.enter_cfunc_scope()
 946             w.putln("static int __Pyx_InitCachedBuiltins(void) {")
 947
 948         w = self.parts['cached_constants']
 949         w.enter_cfunc_scope()
 950         w.putln("")
 951         w.putln("static int __Pyx_InitCachedConstants(void) {")
 952         w.put_declare_refcount_context()
 953         w.put_setup_refcount_context("__Pyx_InitCachedConstants")
 954
 955         w = self.parts['init_globals']
 956         w.enter_cfunc_scope()
 957         w.putln("")
 958         w.putln("static int __Pyx_InitGlobals(void) {")
 959
 960         if not Options.generate_cleanup_code:
 961             del self.parts['cleanup_globals']
 962         else:
 963             w = self.parts['cleanup_globals']
 964             w.enter_cfunc_scope()
 965             w.putln("")
 966             w.putln("static void __Pyx_CleanupGlobals(void) {")
 967
 968         #
 969         # utility_code_def
 970         #
 971         code = self.parts['utility_code_def']
 972         if self.emit_linenums:
 973             code.write('\n#line 1 "cython_utility"\n')
 974         code.putln("")
 975         code.putln("/* Runtime support code */")
 976
 977     def finalize_main_c_code(self):
 978         self.close_global_decls()
 979
 980         #
 981         # utility_code_def
 982         #
 983         code = self.parts['utility_code_def']
 984         code.put(UtilityCode.load_as_string("TypeConversions", "TypeConversion.c")[1])
 985         code.putln("")
 986
 987     def __getitem__(self, key):
 988         return self.parts[key]
 989
 990     #
 991     # Global constants, interned objects, etc.
 992     #
 993     def close_global_decls(self):
 994         # This is called when it is known that no more global declarations will
 995         # declared.
 996         self.generate_const_declarations()
 997         if Options.cache_builtins:
 998             w = self.parts['cached_builtins']
 999             w.putln("return 0;")
1000             if w.label_used(w.error_label):
1001                 w.put_label(w.error_label)
1002                 w.putln("return -1;")
1003             w.putln("}")
1004             w.exit_cfunc_scope()
1005
1006         w = self.parts['cached_constants']
1007         w.put_finish_refcount_context()
1008         w.putln("return 0;")
1009         if w.label_used(w.error_label):
1010             w.put_label(w.error_label)
1011             w.put_finish_refcount_context()
1012             w.putln("return -1;")
1013         w.putln("}")
1014         w.exit_cfunc_scope()
1015
1016         w = self.parts['init_globals']
1017         w.putln("return 0;")
1018         if w.label_used(w.error_label):
1019             w.put_label(w.error_label)
1020             w.putln("return -1;")
1021         w.putln("}")
1022         w.exit_cfunc_scope()
1023
1024         if Options.generate_cleanup_code:
1025             w = self.parts['cleanup_globals']
1026             w.putln("}")
1027             w.exit_cfunc_scope()
1028
1029         if Options.generate_cleanup_code:
1030             w = self.parts['cleanup_module']
1031             w.putln("}")
1032             w.exit_cfunc_scope()
1033
1034     def put_pyobject_decl(self, entry):
1035         self['global_var'].putln("static PyObject *%s;" % entry.cname)
1036
1037     # constant handling at code generation time
1038
1039     def get_cached_constants_writer(self):
1040         return self.parts['cached_constants']
1041
1042     def get_int_const(self, str_value, longness=False):
1043         py_type = longness and 'long' or 'int'
1044         try:
1045             c = self.num_const_index[(str_value, py_type)]
1046         except KeyError:
1047             c = self.new_num_const(str_value, py_type)
1048         return c
1049
1050     def get_float_const(self, str_value, value_code):
1051         try:
1052             c = self.num_const_index[(str_value, 'float')]
1053         except KeyError:
1054             c = self.new_num_const(str_value, 'float', value_code)
1055         return c
1056
1057     def get_py_const(self, type, prefix='', cleanup_level=None):
1058         # create a new Python object constant
1059         const = self.new_py_const(type, prefix)
1060         if cleanup_level is not None \
1061                 and cleanup_level <= Options.generate_cleanup_code:
1062             cleanup_writer = self.parts['cleanup_globals']
1063             cleanup_writer.putln('Py_CLEAR(%s);' % const.cname)
1064         return const
1065
1066     def get_string_const(self, text, py_version=None):
1067         # return a C string constant, creating a new one if necessary
1068         if text.is_unicode:
1069             byte_string = text.utf8encode()
1070         else:
1071             byte_string = text.byteencode()
1072         try:
1073             c = self.string_const_index[byte_string]
1074         except KeyError:
1075             c = self.new_string_const(text, byte_string)
1076         c.add_py_version(py_version)
1077         return c
1078
1079     def get_pyunicode_ptr_const(self, text):
1080         # return a Py_UNICODE[] constant, creating a new one if necessary
1081         assert text.is_unicode
1082         try:
1083             c = self.pyunicode_ptr_const_index[text]
1084         except KeyError:
1085             c = self.pyunicode_ptr_const_index[text] = self.new_const_cname()
1086         return c
1087
1088     def get_py_string_const(self, text, identifier=None,
1089                             is_str=False, unicode_value=None):
1090         # return a Python string constant, creating a new one if necessary
1091         py3str_cstring = None
1092         if is_str and unicode_value is not None \
1093                and unicode_value.utf8encode() != text.byteencode():
1094             py3str_cstring = self.get_string_const(unicode_value, py_version=3)
1095             c_string = self.get_string_const(text, py_version=2)
1096         else:
1097             c_string = self.get_string_const(text)
1098         py_string = c_string.get_py_string_const(
1099             text.encoding, identifier, is_str, py3str_cstring)
1100         return py_string
1101
1102     def get_interned_identifier(self, text):
1103         return self.get_py_string_const(text, identifier=True)
1104
1105     def new_string_const(self, text, byte_string):
1106         cname = self.new_string_const_cname(byte_string)
1107         c = StringConst(cname, text, byte_string)
1108         self.string_const_index[byte_string] = c
1109         return c
1110
1111     def new_num_const(self, value, py_type, value_code=None):
1112         cname = self.new_num_const_cname(value, py_type)
1113         c = NumConst(cname, value, py_type, value_code)
1114         self.num_const_index[(value, py_type)] = c
1115         return c
1116
1117     def new_py_const(self, type, prefix=''):
1118         cname = self.new_const_cname(prefix)
1119         c = PyObjectConst(cname, type)
1120         self.py_constants.append(c)
1121         return c
1122
1123     def new_string_const_cname(self, bytes_value):
1124         # Create a new globally-unique nice name for a C string constant.
1125         value = bytes_value.decode('ASCII', 'ignore')
1126         return self.new_const_cname(value=value)
1127
1128     def new_num_const_cname(self, value, py_type):
1129         if py_type == 'long':
1130             value += 'L'
1131             py_type = 'int'
1132         prefix = Naming.interned_prefixes[py_type]
1133         cname = "%s%s" % (prefix, value)
1134         cname = cname.replace('+', '_').replace('-', 'neg_').replace('.', '_')
1135         return cname
1136
1137     def new_const_cname(self, prefix='', value=''):
1138         value = replace_identifier('_', value)[:32].strip('_')
1139         used = self.const_cnames_used
1140         name_suffix = value
1141         while name_suffix in used:
1142             counter = used[value] = used[value] + 1
1143             name_suffix = '%s_%d' % (value, counter)
1144         used[name_suffix] = 1
1145         if prefix:
1146             prefix = Naming.interned_prefixes[prefix]
1147         else:
1148             prefix = Naming.const_prefix
1149         return "%s%s" % (prefix, name_suffix)
1150
1151     def add_cached_builtin_decl(self, entry):
1152         if entry.is_builtin and entry.is_const:
1153             if self.should_declare(entry.cname, entry):
1154                 self.put_pyobject_decl(entry)
1155                 w = self.parts['cached_builtins']
1156                 condition = None
1157                 if entry.name in non_portable_builtins_map:
1158                     condition, replacement = non_portable_builtins_map[entry.name]
1159                     w.putln('#if %s' % condition)
1160                     self.put_cached_builtin_init(
1161                         entry.pos, StringEncoding.EncodedString(replacement),
1162                         entry.cname)
1163                     w.putln('#else')
1164                 self.put_cached_builtin_init(
1165                     entry.pos, StringEncoding.EncodedString(entry.name),
1166                     entry.cname)
1167                 if condition:
1168                     w.putln('#endif')
1169
1170     def put_cached_builtin_init(self, pos, name, cname):
1171         w = self.parts['cached_builtins']
1172         interned_cname = self.get_interned_identifier(name).cname
1173         self.use_utility_code(
1174             UtilityCode.load_cached("GetBuiltinName", "ObjectHandling.c"))
1175         w.putln('%s = __Pyx_GetBuiltinName(%s); if (!%s) %s' % (
1176             cname,
1177             interned_cname,
1178             cname,
1179             w.error_goto(pos)))
1180
1181     def generate_const_declarations(self):
1182         self.generate_string_constants()
1183         self.generate_num_constants()
1184         self.generate_object_constant_decls()
1185
1186     def generate_object_constant_decls(self):
1187         consts = [ (len(c.cname), c.cname, c)
1188                    for c in self.py_constants ]
1189         consts.sort()
1190         decls_writer = self.parts['decls']
1191         for _, cname, c in consts:
1192             decls_writer.putln(
1193                 "static %s;" % c.type.declaration_code(cname))
1194
1195     def generate_string_constants(self):
1196         c_consts = [ (len(c.cname), c.cname, c)
1197                      for c in self.string_const_index.values() ]
1198         c_consts.sort()
1199         py_strings = []
1200
1201         decls_writer = self.parts['decls']
1202         for _, cname, c in c_consts:
1203             conditional = False
1204             if c.py_versions and (2 not in c.py_versions or 3 not in c.py_versions):
1205                 conditional = True
1206                 decls_writer.putln("#if PY_MAJOR_VERSION %s 3" % (
1207                     (2 in c.py_versions) and '<' or '>='))
1208             decls_writer.putln('static char %s[] = "%s";' % (
1209                 cname, StringEncoding.split_string_literal(c.escaped_value)))
1210             if conditional:
1211                 decls_writer.putln("#endif")
1212             if c.py_strings is not None:
1213                 for py_string in c.py_strings.values():
1214                     py_strings.append((c.cname, len(py_string.cname), py_string))
1215
1216         for c, cname in self.pyunicode_ptr_const_index.items():
1217             utf16_array, utf32_array = StringEncoding.encode_pyunicode_string(c)
1218             if utf16_array:
1219                 # Narrow and wide representations differ
1220                 decls_writer.putln("#ifdef Py_UNICODE_WIDE")
1221             decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf32_array))
1222             if utf16_array:
1223                 decls_writer.putln("#else")
1224                 decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array))
1225                 decls_writer.putln("#endif")
1226
1227         if py_strings:
1228             self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c"))
1229             py_strings.sort()
1230             w = self.parts['pystring_table']
1231             w.putln("")
1232             w.putln("static __Pyx_StringTabEntry %s[] = {" %
1233                                       Naming.stringtab_cname)
1234             for c_cname, _, py_string in py_strings:
1235                 if not py_string.is_str or not py_string.encoding or \
1236                        py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII',
1237                                               'UTF8', 'UTF-8'):
1238                     encoding = '0'
1239                 else:
1240                     encoding = '"%s"' % py_string.encoding.lower()
1241
1242                 decls_writer.putln(
1243                     "static PyObject *%s;" % py_string.cname)
1244                 if py_string.py3str_cstring:
1245                     w.putln("#if PY_MAJOR_VERSION >= 3")
1246                     w.putln(
1247                         "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
1248                         py_string.cname,
1249                         py_string.py3str_cstring.cname,
1250                         py_string.py3str_cstring.cname,
1251                         '0', 1, 0,
1252                         py_string.intern
1253                         ))
1254                     w.putln("#else")
1255                 w.putln(
1256                     "{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
1257                     py_string.cname,
1258                     c_cname,
1259                     c_cname,
1260                     encoding,
1261                     py_string.is_unicode,
1262                     py_string.is_str,
1263                     py_string.intern
1264                     ))
1265                 if py_string.py3str_cstring:
1266                     w.putln("#endif")
1267             w.putln("{0, 0, 0, 0, 0, 0, 0}")
1268             w.putln("};")
1269
1270             init_globals = self.parts['init_globals']
1271             init_globals.putln(
1272                 "if (__Pyx_InitStrings(%s) < 0) %s;" % (
1273                     Naming.stringtab_cname,
1274                     init_globals.error_goto(self.module_pos)))
1275
1276     def generate_num_constants(self):
1277         consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c)
1278                   for c in self.num_const_index.values()]
1279         consts.sort()
1280         decls_writer = self.parts['decls']
1281         init_globals = self.parts['init_globals']
1282         for py_type, _, _, value, value_code, c in consts:
1283             cname = c.cname
1284             decls_writer.putln("static PyObject *%s;" % cname)
1285             if py_type == 'float':
1286                 function = 'PyFloat_FromDouble(%s)'
1287             elif py_type == 'long':
1288                 function = 'PyLong_FromString((char *)"%s", 0, 0)'
1289             elif Utils.long_literal(value):
1290                 function = 'PyInt_FromString((char *)"%s", 0, 0)'
1291             elif len(value.lstrip('-')) > 4:
1292                 function = "PyInt_FromLong(%sL)"
1293             else:
1294                 function = "PyInt_FromLong(%s)"
1295             init_globals.putln('%s = %s; %s' % (
1296                 cname, function % value_code,
1297                 init_globals.error_goto_if_null(cname, self.module_pos)))
1298
1299     # The functions below are there in a transition phase only
1300     # and will be deprecated. They are called from Nodes.BlockNode.
1301     # The copy&paste duplication is intentional in order to be able
1302     # to see quickly how BlockNode worked, until this is replaced.
1303
1304     def should_declare(self, cname, entry):
1305         if cname in self.declared_cnames:
1306             other = self.declared_cnames[cname]
1307             assert str(entry.type) == str(other.type)
1308             assert entry.init == other.init
1309             return False
1310         else:
1311             self.declared_cnames[cname] = entry
1312             return True
1313
1314     #
1315     # File name state
1316     #
1317
1318     def lookup_filename(self, filename):
1319         try:
1320             index = self.filename_table[filename]
1321         except KeyError:
1322             index = len(self.filename_list)
1323             self.filename_list.append(filename)
1324             self.filename_table[filename] = index
1325         return index
1326
1327     def commented_file_contents(self, source_desc):
1328         try:
1329             return self.input_file_contents[source_desc]
1330         except KeyError:
1331             pass
1332         source_file = source_desc.get_lines(encoding='ASCII',
1333                                             error_handling='ignore')
1334         try:
1335             F = [u' * ' + line.rstrip().replace(
1336                     u'*/', u'*[inserted by cython to avoid comment closer]/'
1337                     ).replace(
1338                     u'/*', u'/[inserted by cython to avoid comment start]*'
1339                     )
1340                  for line in source_file]
1341         finally:
1342             if hasattr(source_file, 'close'):
1343                 source_file.close()
1344         if not F: F.append(u'')
1345         self.input_file_contents[source_desc] = F
1346         return F
1347
1348     #
1349     # Utility code state
1350     #
1351
1352     def use_utility_code(self, utility_code):
1353         """
1354         Adds code to the C file. utility_code should
1355         a) implement __eq__/__hash__ for the purpose of knowing whether the same
1356            code has already been included
1357         b) implement put_code, which takes a globalstate instance
1358
1359         See UtilityCode.
1360         """
1361         if utility_code not in self.utility_codes:
1362             self.utility_codes.add(utility_code)
1363             utility_code.put_code(self)
1364
1365
1366 def funccontext_property(name):
1367     attribute_of = operator.attrgetter(name)
1368     def get(self):
1369         return attribute_of(self.funcstate)
1370     def set(self, value):
1371         setattr(self.funcstate, name, value)
1372     return property(get, set)
1373
1374
1375 class CCodeWriter(object):
1376     """
1377     Utility class to output C code.
1378
1379     When creating an insertion point one must care about the state that is
1380     kept:
1381     - formatting state (level, bol) is cloned and used in insertion points
1382       as well
1383     - labels, temps, exc_vars: One must construct a scope in which these can
1384       exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for
1385       sanity checking and forward compatabilty). Created insertion points
1386       looses this scope and cannot access it.
1387     - marker: Not copied to insertion point
1388     - filename_table, filename_list, input_file_contents: All codewriters
1389       coming from the same root share the same instances simultaneously.
1390     """
1391
1392     # f                   file            output file
1393     # buffer              StringIOTree
1394
1395     # level               int             indentation level
1396     # bol                 bool            beginning of line?
1397     # marker              string          comment to emit before next line
1398     # funcstate           FunctionState   contains state local to a C function used for code
1399     #                                     generation (labels and temps state etc.)
1400     # globalstate         GlobalState     contains state global for a C file (input file info,
1401     #                                     utility code, declared constants etc.)
1402     # emit_linenums       boolean         whether or not to write #line pragmas
1403     #
1404     # c_line_in_traceback boolean         append the c file and line number to the traceback for exceptions
1405     #
1406     # pyclass_stack       list            used during recursive code generation to pass information
1407     #                                     about the current class one is in
1408
1409     globalstate = None
1410
1411     def __init__(self, create_from=None, buffer=None, copy_formatting=False, emit_linenums=None, c_line_in_traceback=True):
1412         if buffer is None: buffer = StringIOTree()
1413         self.buffer = buffer
1414         self.marker = None
1415         self.last_marker_line = 0
1416         self.source_desc = ""
1417         self.pyclass_stack = []
1418
1419         self.funcstate = None
1420         self.level = 0
1421         self.call_level = 0
1422         self.bol = 1
1423
1424         if create_from is not None:
1425             # Use same global state
1426             self.globalstate = create_from.globalstate
1427             self.funcstate = create_from.funcstate
1428             # Clone formatting state
1429             if copy_formatting:
1430                 self.level = create_from.level
1431                 self.bol = create_from.bol
1432                 self.call_level = create_from.call_level
1433
1434         if emit_linenums is None and self.globalstate:
1435             self.emit_linenums = self.globalstate.emit_linenums
1436         else:
1437             self.emit_linenums = emit_linenums
1438         self.c_line_in_traceback = c_line_in_traceback
1439
1440     def create_new(self, create_from, buffer, copy_formatting):
1441         # polymorphic constructor -- very slightly more versatile
1442         # than using __class__
1443         result = CCodeWriter(create_from, buffer, copy_formatting,
1444                              c_line_in_traceback=self.c_line_in_traceback)
1445         return result
1446
1447     def copyto(self, f):
1448         self.buffer.copyto(f)
1449
1450     def getvalue(self):
1451         return self.buffer.getvalue()
1452
1453     def write(self, s):
1454         # also put invalid markers (lineno 0), to indicate that those lines
1455         # have no Cython source code correspondence
1456         if self.marker is None:
1457             cython_lineno = self.last_marker_line
1458         else:
1459             cython_lineno = self.marker[0]
1460
1461         self.buffer.markers.extend([cython_lineno] * s.count('\n'))
1462         self.buffer.write(s)
1463
1464     def insertion_point(self):
1465         other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
1466         return other
1467
1468     def new_writer(self):
1469         """
1470         Creates a new CCodeWriter connected to the same global state, which
1471         can later be inserted using insert.
1472         """
1473         return CCodeWriter(create_from=self, c_line_in_traceback=self.c_line_in_traceback)
1474
1475     def insert(self, writer):
1476         """
1477         Inserts the contents of another code writer (created with
1478         the same global state) in the current location.
1479
1480         It is ok to write to the inserted writer also after insertion.
1481         """
1482         assert writer.globalstate is self.globalstate
1483         self.buffer.insert(writer.buffer)
1484
1485     # Properties delegated to function scope
1486     label_counter = funccontext_property("label_counter")
1487     return_label = funccontext_property("return_label")
1488     error_label = funccontext_property("error_label")
1489     labels_used = funccontext_property("labels_used")
1490     continue_label = funccontext_property("continue_label")
1491     break_label = funccontext_property("break_label")
1492     return_from_error_cleanup_label = funccontext_property("return_from_error_cleanup_label")
1493     yield_labels = funccontext_property("yield_labels")
1494
1495     # Functions delegated to function scope
1496     def new_label(self, name=None):    return self.funcstate.new_label(name)
1497     def new_error_label(self):         return self.funcstate.new_error_label()
1498     def new_yield_label(self):         return self.funcstate.new_yield_label()
1499     def get_loop_labels(self):         return self.funcstate.get_loop_labels()
1500     def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels)
1501     def new_loop_labels(self):         return self.funcstate.new_loop_labels()
1502     def get_all_labels(self):          return self.funcstate.get_all_labels()
1503     def set_all_labels(self, labels):  return self.funcstate.set_all_labels(labels)
1504     def all_new_labels(self):          return self.funcstate.all_new_labels()
1505     def use_label(self, lbl):          return self.funcstate.use_label(lbl)
1506     def label_used(self, lbl):         return self.funcstate.label_used(lbl)
1507
1508
1509     def enter_cfunc_scope(self):
1510         self.funcstate = FunctionState(self)
1511
1512     def exit_cfunc_scope(self):
1513         self.funcstate = None
1514
1515     # constant handling
1516
1517     def get_py_int(self, str_value, longness):
1518         return self.globalstate.get_int_const(str_value, longness).cname
1519
1520     def get_py_float(self, str_value, value_code):
1521         return self.globalstate.get_float_const(str_value, value_code).cname
1522
1523     def get_py_const(self, type, prefix='', cleanup_level=None):
1524         return self.globalstate.get_py_const(type, prefix, cleanup_level).cname
1525
1526     def get_string_const(self, text):
1527         return self.globalstate.get_string_const(text).cname
1528
1529     def get_pyunicode_ptr_const(self, text):
1530         return self.globalstate.get_pyunicode_ptr_const(text)
1531
1532     def get_py_string_const(self, text, identifier=None,
1533                             is_str=False, unicode_value=None):
1534         return self.globalstate.get_py_string_const(
1535             text, identifier, is_str, unicode_value).cname
1536
1537     def get_argument_default_const(self, type):
1538         return self.globalstate.get_py_const(type).cname
1539
1540     def intern(self, text):
1541         return self.get_py_string_const(text)
1542
1543     def intern_identifier(self, text):
1544         return self.get_py_string_const(text, identifier=True)
1545
1546     def get_cached_constants_writer(self):
1547         return self.globalstate.get_cached_constants_writer()
1548
1549     # code generation
1550
1551     def putln(self, code="", safe=False):
1552         if self.marker and self.bol:
1553             self.emit_marker()
1554         if self.emit_linenums and self.last_marker_line != 0:
1555             self.write('\n#line %s "%s"\n' % (self.last_marker_line, self.source_desc))
1556
1557         if code:
1558             if safe:
1559                 self.put_safe(code)
1560             else:
1561                 self.put(code)
1562         self.write("\n")
1563         self.bol = 1
1564
1565     def emit_marker(self):
1566         self.write("\n")
1567         self.indent()
1568         self.write("/* %s */\n" % self.marker[1])
1569         if (self.funcstate and self.funcstate.can_trace
1570                 and self.globalstate.directives['linetrace']):
1571             self.indent()
1572             self.write('__Pyx_TraceLine(%d)\n' % self.marker[0])
1573         self.last_marker_line = self.marker[0]
1574         self.marker = None
1575
1576     def put_safe(self, code):
1577         # put code, but ignore {}
1578         self.write(code)
1579         self.bol = 0
1580
1581     def put_or_include(self, code, name):
1582         include_dir = self.globalstate.common_utility_include_dir
1583         if include_dir and len(code) > 1024:
1584             include_file = "%s_%s.h" % (
1585                 name, hashlib.md5(code.encode('utf8')).hexdigest())
1586             path = os.path.join(include_dir, include_file)
1587             if not os.path.exists(path):
1588                 tmp_path = '%s.tmp%s' % (path, os.getpid())
1589                 f = Utils.open_new_file(tmp_path)
1590                 try:
1591                     f.write(code)
1592                 finally:
1593                     f.close()
1594                 os.rename(tmp_path, path)
1595             code = '#include "%s"\n' % path
1596         self.put(code)
1597
1598     def put(self, code):
1599         if is_self_assignment(code):
1600             return
1601         fix_indent = False
1602         if "{" in code:
1603             dl = code.count("{")
1604         else:
1605             dl = 0
1606         if "}" in code:
1607             dl -= code.count("}")
1608             if dl < 0:
1609                 self.level += dl
1610             elif dl == 0 and code[0] == "}":
1611                 # special cases like "} else {" need a temporary dedent
1612                 fix_indent = True
1613                 self.level -= 1
1614         if self.bol:
1615             self.indent()
1616         self.write(code)
1617         self.bol = 0
1618         if dl > 0:
1619             self.level += dl
1620         elif fix_indent:
1621             self.level += 1
1622
1623     def putln_tempita(self, code, **context):
1624         from Cython.Tempita import sub
1625         self.putln(sub(code, **context))
1626
1627     def put_tempita(self, code, **context):
1628         from Cython.Tempita import sub
1629         self.put(sub(code, **context))
1630
1631     def increase_indent(self):
1632         self.level += 1
1633
1634     def decrease_indent(self):
1635         self.level -= 1
1636
1637     def begin_block(self):
1638         self.putln("{")
1639         self.increase_indent()
1640
1641     def end_block(self):
1642         self.decrease_indent()
1643         self.putln("}")
1644
1645     def indent(self):
1646         self.write("  " * self.level)
1647
1648     def get_py_version_hex(self, pyversion):
1649         return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]
1650
1651     def mark_pos(self, pos):
1652         if pos is None:
1653             return
1654         source_desc, line, col = pos
1655         if self.last_marker_line == line:
1656             return
1657         assert isinstance(source_desc, SourceDescriptor)
1658         contents = self.globalstate.commented_file_contents(source_desc)
1659         lines = contents[max(0, line-3):line]  # line numbers start at 1
1660         lines[-1] += u'             # <<<<<<<<<<<<<<'
1661         lines += contents[line:line+2]
1662
1663         marker = u'"%s":%d\n%s\n' % (
1664             source_desc.get_escaped_description(), line, u'\n'.join(lines))
1665         self.marker = (line, marker)
1666         if self.emit_linenums:
1667             self.source_desc = source_desc.get_escaped_description()
1668
1669     def put_label(self, lbl):
1670         if lbl in self.funcstate.labels_used:
1671             self.putln("%s:;" % lbl)
1672
1673     def put_goto(self, lbl):
1674         self.funcstate.use_label(lbl)
1675         self.putln("goto %s;" % lbl)
1676
1677     def put_var_declaration(self, entry, storage_class="",
1678                             dll_linkage=None, definition=True):
1679         #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
1680         if entry.visibility == 'private' and not (definition or entry.defined_in_pxd):
1681             #print "...private and not definition, skipping", entry.cname ###
1682             return
1683         if entry.visibility == "private" and not entry.used:
1684             #print "...private and not used, skipping", entry.cname ###
1685             return
1686         if storage_class:
1687             self.put("%s " % storage_class)
1688         if not entry.cf_used:
1689             self.put('CYTHON_UNUSED ')
1690         self.put(entry.type.declaration_code(
1691             entry.cname, dll_linkage=dll_linkage))
1692         if entry.init is not None:
1693             self.put_safe(" = %s" % entry.type.literal_code(entry.init))
1694         elif entry.type.is_pyobject:
1695             self.put(" = NULL")
1696         self.putln(";")
1697
1698     def put_temp_declarations(self, func_context):
1699         for name, type, manage_ref, static in func_context.temps_allocated:
1700             decl = type.declaration_code(name)
1701             if type.is_pyobject:
1702                 self.putln("%s = NULL;" % decl)
1703             elif type.is_memoryviewslice:
1704                 import MemoryView
1705                 self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init))
1706             else:
1707                 self.putln("%s%s;" % (static and "static " or "", decl))
1708
1709         if func_context.should_declare_error_indicator:
1710             if self.funcstate.uses_error_indicator:
1711                 unused = ''
1712             else:
1713                 unused = 'CYTHON_UNUSED '
1714             # Initialize these variables to silence compiler warnings
1715             self.putln("%sint %s = 0;" % (unused, Naming.lineno_cname))
1716             self.putln("%sconst char *%s = NULL;" % (unused, Naming.filename_cname))
1717             self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname))
1718
1719     def put_h_guard(self, guard):
1720         self.putln("#ifndef %s" % guard)
1721         self.putln("#define %s" % guard)
1722
1723     def unlikely(self, cond):
1724         if Options.gcc_branch_hints:
1725             return 'unlikely(%s)' % cond
1726         else:
1727             return cond
1728
1729     def build_function_modifiers(self, modifiers, mapper=modifier_output_mapper):
1730         if not modifiers:
1731             return ''
1732         return '%s ' % ' '.join([mapper(m,m) for m in modifiers])
1733
1734     # Python objects and reference counting
1735
1736     def entry_as_pyobject(self, entry):
1737         type = entry.type
1738         if (not entry.is_self_arg and not entry.type.is_complete()
1739             or entry.type.is_extension_type):
1740             return "(PyObject *)" + entry.cname
1741         else:
1742             return entry.cname
1743
1744     def as_pyobject(self, cname, type):
1745         from PyrexTypes import py_object_type, typecast
1746         return typecast(py_object_type, type, cname)
1747
1748     def put_gotref(self, cname):
1749         self.putln("__Pyx_GOTREF(%s);" % cname)
1750
1751     def put_giveref(self, cname):
1752         self.putln("__Pyx_GIVEREF(%s);" % cname)
1753
1754     def put_xgiveref(self, cname):
1755         self.putln("__Pyx_XGIVEREF(%s);" % cname)
1756
1757     def put_xgotref(self, cname):
1758         self.putln("__Pyx_XGOTREF(%s);" % cname)
1759
1760     def put_incref(self, cname, type, nanny=True):
1761         if nanny:
1762             self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type))
1763         else:
1764             self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type))
1765
1766     def put_decref(self, cname, type, nanny=True):
1767         self._put_decref(cname, type, nanny, null_check=False, clear=False)
1768
1769     def put_var_gotref(self, entry):
1770         if entry.type.is_pyobject:
1771             self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry))
1772
1773     def put_var_giveref(self, entry):
1774         if entry.type.is_pyobject:
1775             self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry))
1776
1777     def put_var_xgotref(self, entry):
1778         if entry.type.is_pyobject:
1779             self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry))
1780
1781     def put_var_xgiveref(self, entry):
1782         if entry.type.is_pyobject:
1783             self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry))
1784
1785     def put_var_incref(self, entry):
1786         if entry.type.is_pyobject:
1787             self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry))
1788
1789     def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False):
1790         self._put_decref(cname, type, nanny, null_check=False,
1791                          clear=True, clear_before_decref=clear_before_decref)
1792
1793     def put_xdecref(self, cname, type, nanny=True, have_gil=True):
1794         self._put_decref(cname, type, nanny, null_check=True,
1795                          have_gil=have_gil, clear=False)
1796
1797     def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False):
1798         self._put_decref(cname, type, nanny, null_check=True,
1799                          clear=True, clear_before_decref=clear_before_decref)
1800
1801     def _put_decref(self, cname, type, nanny=True, null_check=False,
1802                     have_gil=True, clear=False, clear_before_decref=False):
1803         if type.is_memoryviewslice:
1804             self.put_xdecref_memoryviewslice(cname, have_gil=have_gil)
1805             return
1806
1807         prefix = nanny and '__Pyx' or 'Py'
1808         X = null_check and 'X' or ''
1809
1810         if clear:
1811             if clear_before_decref:
1812                 if not nanny:
1813                     X = ''  # CPython doesn't have a Py_XCLEAR()
1814                 self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname))
1815             else:
1816                 self.putln("%s_%sDECREF(%s); %s = 0;" % (
1817                     prefix, X, self.as_pyobject(cname, type), cname))
1818         else:
1819             self.putln("%s_%sDECREF(%s);" % (
1820                 prefix, X, self.as_pyobject(cname, type)))
1821
1822     def put_decref_set(self, cname, rhs_cname):
1823         self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname))
1824
1825     def put_xdecref_set(self, cname, rhs_cname):
1826         self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname))
1827
1828     def put_var_decref(self, entry):
1829         if entry.type.is_pyobject:
1830             self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
1831
1832     def put_var_xdecref(self, entry):
1833         if entry.type.is_pyobject:
1834             self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
1835
1836     def put_var_decref_clear(self, entry):
1837         self._put_var_decref_clear(entry, null_check=False)
1838
1839     def put_var_xdecref_clear(self, entry):
1840         self._put_var_decref_clear(entry, null_check=True)
1841
1842     def _put_var_decref_clear(self, entry, null_check):
1843         if entry.type.is_pyobject:
1844             if entry.in_closure:
1845                 # reset before DECREF to make sure closure state is
1846                 # consistent during call to DECREF()
1847                 self.putln("__Pyx_%sCLEAR(%s);" % (
1848                     null_check and 'X' or '',
1849                     entry.cname))
1850             else:
1851                 self.putln("__Pyx_%sDECREF(%s); %s = 0;" % (
1852                     null_check and 'X' or '',
1853                     self.entry_as_pyobject(entry),
1854                     entry.cname))
1855
1856     def put_var_decrefs(self, entries, used_only = 0):
1857         for entry in entries:
1858             if not used_only or entry.used:
1859                 if entry.xdecref_cleanup:
1860                     self.put_var_xdecref(entry)
1861                 else:
1862                     self.put_var_decref(entry)
1863
1864     def put_var_xdecrefs(self, entries):
1865         for entry in entries:
1866             self.put_var_xdecref(entry)
1867
1868     def put_var_xdecrefs_clear(self, entries):
1869         for entry in entries:
1870             self.put_var_xdecref_clear(entry)
1871
1872     def put_incref_memoryviewslice(self, slice_cname, have_gil=False):
1873         import MemoryView
1874         self.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
1875         self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))
1876
1877     def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False):
1878         import MemoryView
1879         self.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
1880         self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))
1881
1882     def put_xgiveref_memoryviewslice(self, slice_cname):
1883         self.put_xgiveref("%s.memview" % slice_cname)
1884
1885     def put_init_to_py_none(self, cname, type, nanny=True):
1886         from PyrexTypes import py_object_type, typecast
1887         py_none = typecast(type, py_object_type, "Py_None")
1888         if nanny:
1889             self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none))
1890         else:
1891             self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none))
1892
1893     def put_init_var_to_py_none(self, entry, template = "%s", nanny=True):
1894         code = template % entry.cname
1895         #if entry.type.is_extension_type:
1896         #    code = "((PyObject*)%s)" % code
1897         self.put_init_to_py_none(code, entry.type, nanny)
1898         if entry.in_closure:
1899             self.put_giveref('Py_None')
1900
1901     def put_pymethoddef(self, entry, term, allow_skip=True):
1902         if entry.is_special or entry.name == '__getattribute__':
1903             if entry.name not in ['__cinit__', '__dealloc__', '__richcmp__', '__next__', '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__', '__getcharbuffer__', '__getbuffer__', '__releasebuffer__']:
1904                 if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']:
1905                     pass
1906                 # Python's typeobject.c will automatically fill in our slot
1907                 # in add_operators() (called by PyType_Ready) with a value
1908                 # that's better than ours.
1909                 elif allow_skip:
1910                     return
1911         from TypeSlots import method_coexist
1912         if entry.doc:
1913             doc_code = entry.doc_cname
1914         else:
1915             doc_code = 0
1916         method_flags = entry.signature.method_flags()
1917         if method_flags:
1918             if entry.is_special:
1919                 method_flags += [method_coexist]
1920             self.putln(
1921                 '{__Pyx_NAMESTR("%s"), (PyCFunction)%s, %s, __Pyx_DOCSTR(%s)}%s' % (
1922                     entry.name,
1923                     entry.func_cname,
1924                     "|".join(method_flags),
1925                     doc_code,
1926                     term))
1927
1928     # GIL methods
1929
1930     def put_ensure_gil(self, declare_gilstate=True, variable=None):
1931         """
1932         Acquire the GIL. The generated code is safe even when no PyThreadState
1933         has been allocated for this thread (for threads not initialized by
1934         using the Python API). Additionally, the code generated by this method
1935         may be called recursively.
1936         """
1937         self.globalstate.use_utility_code(
1938             UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
1939         self.putln("#ifdef WITH_THREAD")
1940         if not variable:
1941             variable = '__pyx_gilstate_save'
1942             if declare_gilstate:
1943                 self.put("PyGILState_STATE ")
1944         self.putln("%s = PyGILState_Ensure();" % variable)
1945         self.putln("#endif")
1946
1947     def put_release_ensured_gil(self, variable=None):
1948         """
1949         Releases the GIL, corresponds to `put_ensure_gil`.
1950         """
1951         if not variable:
1952             variable = '__pyx_gilstate_save'
1953         self.putln("#ifdef WITH_THREAD")
1954         self.putln("PyGILState_Release(%s);" % variable)
1955         self.putln("#endif")
1956
1957     def put_acquire_gil(self, variable=None):
1958         """
1959         Acquire the GIL. The thread's thread state must have been initialized
1960         by a previous `put_release_gil`
1961         """
1962         self.putln("#ifdef WITH_THREAD")
1963         if variable:
1964             self.putln('_save = %s;' % variable)
1965         self.putln("Py_BLOCK_THREADS")
1966         self.putln("#endif")
1967
1968     def put_release_gil(self, variable=None):
1969         "Release the GIL, corresponds to `put_acquire_gil`."
1970         self.putln("#ifdef WITH_THREAD")
1971         self.putln("PyThreadState *_save;")
1972         self.putln("Py_UNBLOCK_THREADS")
1973         if variable:
1974             self.putln('%s = _save;' % variable)
1975         self.putln("#endif")
1976
1977     def declare_gilstate(self):
1978         self.putln("#ifdef WITH_THREAD")
1979         self.putln("PyGILState_STATE __pyx_gilstate_save;")
1980         self.putln("#endif")
1981
1982     # error handling
1983
1984     def put_error_if_neg(self, pos, value):
1985 #        return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))  # TODO this path is almost _never_ taken, yet this macro makes is slower!
1986         return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))
1987
1988     def put_error_if_unbound(self, pos, entry, in_nogil_context=False):
1989         import ExprNodes
1990         if entry.from_closure:
1991             func = '__Pyx_RaiseClosureNameError'
1992             self.globalstate.use_utility_code(
1993                 ExprNodes.raise_closure_name_error_utility_code)
1994         elif entry.type.is_memoryviewslice and in_nogil_context:
1995             func = '__Pyx_RaiseUnboundMemoryviewSliceNogil'
1996             self.globalstate.use_utility_code(
1997                 ExprNodes.raise_unbound_memoryview_utility_code_nogil)
1998         else:
1999             func = '__Pyx_RaiseUnboundLocalError'
2000             self.globalstate.use_utility_code(
2001                 ExprNodes.raise_unbound_local_error_utility_code)
2002
2003         self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % (
2004                                 entry.type.check_for_null_code(entry.cname),
2005                                 func,
2006                                 entry.name,
2007                                 self.error_goto(pos)))
2008
2009     def set_error_info(self, pos, used=False):
2010         self.funcstate.should_declare_error_indicator = True
2011         if used:
2012             self.funcstate.uses_error_indicator = True
2013         if self.c_line_in_traceback:
2014             cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
2015         else:
2016             cinfo = ""
2017
2018         return "%s = %s[%s]; %s = %s;%s" % (
2019             Naming.filename_cname,
2020             Naming.filetable_cname,
2021             self.lookup_filename(pos[0]),
2022             Naming.lineno_cname,
2023             pos[1],
2024             cinfo)
2025
2026     def error_goto(self, pos):
2027         lbl = self.funcstate.error_label
2028         self.funcstate.use_label(lbl)
2029         return "{%s goto %s;}" % (
2030             self.set_error_info(pos),
2031             lbl)
2032
2033     def error_goto_if(self, cond, pos):
2034         return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos))
2035
2036     def error_goto_if_null(self, cname, pos):
2037         return self.error_goto_if("!%s" % cname, pos)
2038
2039     def error_goto_if_neg(self, cname, pos):
2040         return self.error_goto_if("%s < 0" % cname, pos)
2041
2042     def error_goto_if_PyErr(self, pos):
2043         return self.error_goto_if("PyErr_Occurred()", pos)
2044
2045     def lookup_filename(self, filename):
2046         return self.globalstate.lookup_filename(filename)
2047
2048     def put_declare_refcount_context(self):
2049         self.putln('__Pyx_RefNannyDeclarations')
2050
2051     def put_setup_refcount_context(self, name, acquire_gil=False):
2052         if acquire_gil:
2053             self.globalstate.use_utility_code(
2054                 UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
2055         self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0))
2056
2057     def put_finish_refcount_context(self):
2058         self.putln("__Pyx_RefNannyFinishContext();")
2059
2060     def put_add_traceback(self, qualified_name):
2061         """
2062         Build a Python traceback for propagating exceptions.
2063
2064         qualified_name should be the qualified name of the function.
2065         """
2066         format_tuple = (
2067             qualified_name,
2068             Naming.clineno_cname,
2069             Naming.lineno_cname,
2070             Naming.filename_cname,
2071         )
2072         self.funcstate.uses_error_indicator = True
2073         self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple)
2074
2075     def put_unraisable(self, qualified_name):
2076         """
2077         Generate code to print a Python warning for an unraisable exception.
2078
2079         qualified_name should be the qualified name of the function.
2080         """
2081         format_tuple = (
2082             qualified_name,
2083             Naming.clineno_cname,
2084             Naming.lineno_cname,
2085             Naming.filename_cname,
2086             int(self.globalstate.directives['unraisable_tracebacks'])
2087         )
2088         self.funcstate.uses_error_indicator = True
2089         self.putln('__Pyx_WriteUnraisable("%s", %s, %s, %s, %s);' % format_tuple)
2090         self.globalstate.use_utility_code(
2091             UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c"))
2092
2093     def put_trace_declarations(self):
2094         self.putln('__Pyx_TraceDeclarations')
2095
2096     def put_trace_call(self, name, pos):
2097         self.putln('__Pyx_TraceCall("%s", %s[%s], %s);' % (name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1]))
2098
2099     def put_trace_exception(self):
2100         self.putln("__Pyx_TraceException();")
2101
2102     def put_trace_return(self, retvalue_cname):
2103         self.putln("__Pyx_TraceReturn(%s);" % retvalue_cname)
2104
2105     def putln_openmp(self, string):
2106         self.putln("#ifdef _OPENMP")
2107         self.putln(string)
2108         self.putln("#endif /* _OPENMP */")
2109
2110     def undef_builtin_expect(self, cond):
2111         """
2112         Redefine the macros likely() and unlikely to no-ops, depending on
2113         condition 'cond'
2114         """
2115         self.putln("#if %s" % cond)
2116         self.putln("    #undef likely")
2117         self.putln("    #undef unlikely")
2118         self.putln("    #define likely(x)   (x)")
2119         self.putln("    #define unlikely(x) (x)")
2120         self.putln("#endif")
2121
2122     def redef_builtin_expect(self, cond):
2123         self.putln("#if %s" % cond)
2124         self.putln("    #undef likely")
2125         self.putln("    #undef unlikely")
2126         self.putln("    #define likely(x)   __builtin_expect(!!(x), 1)")
2127         self.putln("    #define unlikely(x) __builtin_expect(!!(x), 0)")
2128         self.putln("#endif")
2129
2130 class PyrexCodeWriter(object):
2131     # f                file      output file
2132     # level            int       indentation level
2133
2134     def __init__(self, outfile_name):
2135         self.f = Utils.open_new_file(outfile_name)
2136         self.level = 0
2137
2138     def putln(self, code):
2139         self.f.write("%s%s\n" % (" " * self.level, code))
2140
2141     def indent(self):
2142         self.level += 1
2143
2144     def dedent(self):
2145         self.level -= 1
2146
2147 class PyxCodeWriter(object):
2148     """
2149     Can be used for writing out some Cython code. To use the indenter
2150     functionality, the Cython.Compiler.Importer module will have to be used
2151     to load the code to support python 2.4
2152     """
2153
2154     def __init__(self, buffer=None, indent_level=0, context=None, encoding='ascii'):
2155         self.buffer = buffer or StringIOTree()
2156         self.level = indent_level
2157         self.context = context
2158         self.encoding = encoding
2159
2160     def indent(self, levels=1):
2161         self.level += levels
2162         return True
2163
2164     def dedent(self, levels=1):
2165         self.level -= levels
2166
2167     def indenter(self, line):
2168         """
2169         Instead of
2170
2171             with pyx_code.indenter("for i in range(10):"):
2172                 pyx_code.putln("print i")
2173
2174         write
2175
2176             if pyx_code.indenter("for i in range(10);"):
2177                 pyx_code.putln("print i")
2178                 pyx_code.dedent()
2179         """
2180         self.putln(line)
2181         self.indent()
2182         return True
2183
2184     def getvalue(self):
2185         result = self.buffer.getvalue()
2186         if not isinstance(result, unicode):
2187             result = result.decode(self.encoding)
2188
2189         return result
2190
2191     def putln(self, line, context=None):
2192         context = context or self.context
2193         if context:
2194             line = sub_tempita(line, context)
2195         self._putln(line)
2196
2197     def _putln(self, line):
2198         self.buffer.write("%s%s\n" % (self.level * "    ", line))
2199
2200     def put_chunk(self, chunk, context=None):
2201         context = context or self.context
2202         if context:
2203             chunk = sub_tempita(chunk, context)
2204
2205         chunk = textwrap.dedent(chunk)
2206         for line in chunk.splitlines():
2207             self._putln(line)
2208
2209     def insertion_point(self):
2210         return PyxCodeWriter(self.buffer.insertion_point(), self.level,
2211                              self.context)
2212
2213     def named_insertion_point(self, name):
2214         setattr(self, name, self.insertion_point())
2215
2216
2217 class ClosureTempAllocator(object):
2218     def __init__(self, klass):
2219         self.klass = klass
2220         self.temps_allocated = {}
2221         self.temps_free = {}
2222         self.temps_count = 0
2223
2224     def reset(self):
2225         for type, cnames in self.temps_allocated.items():
2226             self.temps_free[type] = list(cnames)
2227
2228     def allocate_temp(self, type):
2229         if not type in self.temps_allocated:
2230             self.temps_allocated[type] = []
2231             self.temps_free[type] = []
2232         elif self.temps_free[type]:
2233             return self.temps_free[type].pop(0)
2234         cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count)
2235         self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_cdef=True)
2236         self.temps_allocated[type].append(cname)
2237         self.temps_count += 1
2238         return cname