third_party/cython/src/Cython/Compiler/Pipeline.py

   1 import itertools
   2 from time import time
   3
   4 import Errors
   5 import DebugFlags
   6 import Options
   7 from Visitor import CythonTransform
   8 from Errors import CompileError, InternalError, AbortError
   9 import Naming
  10
  11 #
  12 # Really small pipeline stages
  13 #
  14 def dumptree(t):
  15     # For quick debugging in pipelines
  16     print t.dump()
  17     return t
  18
  19 def abort_on_errors(node):
  20     # Stop the pipeline if there are any errors.
  21     if Errors.num_errors != 0:
  22         raise AbortError("pipeline break")
  23     return node
  24
  25 def parse_stage_factory(context):
  26     def parse(compsrc):
  27         source_desc = compsrc.source_desc
  28         full_module_name = compsrc.full_module_name
  29         initial_pos = (source_desc, 1, 0)
  30         saved_cimport_from_pyx, Options.cimport_from_pyx = Options.cimport_from_pyx, False
  31         scope = context.find_module(full_module_name, pos = initial_pos, need_pxd = 0,
  32                                     check_module_name = not Options.embed)
  33         Options.cimport_from_pyx = saved_cimport_from_pyx
  34         tree = context.parse(source_desc, scope, pxd = 0, full_module_name = full_module_name)
  35         tree.compilation_source = compsrc
  36         tree.scope = scope
  37         tree.is_pxd = False
  38         return tree
  39     return parse
  40
  41 def parse_pxd_stage_factory(context, scope, module_name):
  42     def parse(source_desc):
  43         tree = context.parse(source_desc, scope, pxd=True,
  44                              full_module_name=module_name)
  45         tree.scope = scope
  46         tree.is_pxd = True
  47         return tree
  48     return parse
  49
  50 def generate_pyx_code_stage_factory(options, result):
  51     def generate_pyx_code_stage(module_node):
  52         module_node.process_implementation(options, result)
  53         result.compilation_source = module_node.compilation_source
  54         return result
  55     return generate_pyx_code_stage
  56
  57 def inject_pxd_code_stage_factory(context):
  58     def inject_pxd_code_stage(module_node):
  59         from textwrap import dedent
  60         stats = module_node.body.stats
  61         for name, (statlistnode, scope) in context.pxds.iteritems():
  62             module_node.merge_in(statlistnode, scope)
  63         return module_node
  64     return inject_pxd_code_stage
  65
  66 def use_utility_code_definitions(scope, target, seen=None):
  67     if seen is None:
  68         seen = set()
  69
  70     for entry in scope.entries.itervalues():
  71         if entry in seen:
  72             continue
  73
  74         seen.add(entry)
  75         if entry.used and entry.utility_code_definition:
  76             target.use_utility_code(entry.utility_code_definition)
  77             for required_utility in entry.utility_code_definition.requires:
  78                 target.use_utility_code(required_utility)
  79         elif entry.as_module:
  80             use_utility_code_definitions(entry.as_module, target, seen)
  81
  82 def inject_utility_code_stage_factory(context):
  83     def inject_utility_code_stage(module_node):
  84         use_utility_code_definitions(context.cython_scope, module_node.scope)
  85         added = []
  86         # Note: the list might be extended inside the loop (if some utility code
  87         # pulls in other utility code, explicitly or implicitly)
  88         for utilcode in module_node.scope.utility_code_list:
  89             if utilcode in added: continue
  90             added.append(utilcode)
  91             if utilcode.requires:
  92                 for dep in utilcode.requires:
  93                     if not dep in added and not dep in module_node.scope.utility_code_list:
  94                         module_node.scope.utility_code_list.append(dep)
  95             tree = utilcode.get_tree()
  96             if tree:
  97                 module_node.merge_in(tree.body, tree.scope, merge_scope=True)
  98         return module_node
  99     return inject_utility_code_stage
 100
 101 class UseUtilityCodeDefinitions(CythonTransform):
 102     # Temporary hack to use any utility code in nodes' "utility_code_definitions".
 103     # This should be moved to the code generation phase of the relevant nodes once
 104     # it is safe to generate CythonUtilityCode at code generation time.
 105     def __call__(self, node):
 106         self.scope = node.scope
 107         return super(UseUtilityCodeDefinitions, self).__call__(node)
 108
 109     def process_entry(self, entry):
 110         if entry:
 111             for utility_code in (entry.utility_code, entry.utility_code_definition):
 112                 if utility_code:
 113                     self.scope.use_utility_code(utility_code)
 114
 115     def visit_AttributeNode(self, node):
 116         self.process_entry(node.entry)
 117         return node
 118
 119     def visit_NameNode(self, node):
 120         self.process_entry(node.entry)
 121         self.process_entry(node.type_entry)
 122         return node
 123
 124 #
 125 # Pipeline factories
 126 #
 127
 128 def create_pipeline(context, mode, exclude_classes=()):
 129     assert mode in ('pyx', 'py', 'pxd')
 130     from Visitor import PrintTree
 131     from ParseTreeTransforms import WithTransform, NormalizeTree, PostParse, PxdPostParse
 132     from ParseTreeTransforms import ForwardDeclareTypes, AnalyseDeclarationsTransform
 133     from ParseTreeTransforms import AnalyseExpressionsTransform, FindInvalidUseOfFusedTypes
 134     from ParseTreeTransforms import CreateClosureClasses, MarkClosureVisitor, DecoratorTransform
 135     from ParseTreeTransforms import InterpretCompilerDirectives, TransformBuiltinMethods
 136     from ParseTreeTransforms import ExpandInplaceOperators, ParallelRangeTransform
 137     from ParseTreeTransforms import CalculateQualifiedNamesTransform
 138     from TypeInference import MarkParallelAssignments, MarkOverflowingArithmetic
 139     from ParseTreeTransforms import AdjustDefByDirectives, AlignFunctionDefinitions
 140     from ParseTreeTransforms import RemoveUnreachableCode, GilCheck
 141     from FlowControl import ControlFlowAnalysis
 142     from AnalysedTreeTransforms import AutoTestDictTransform
 143     from AutoDocTransforms import EmbedSignature
 144     from Optimize import FlattenInListTransform, SwitchTransform, IterationTransform
 145     from Optimize import EarlyReplaceBuiltinCalls, OptimizeBuiltinCalls
 146     from Optimize import InlineDefNodeCalls
 147     from Optimize import ConstantFolding, FinalOptimizePhase
 148     from Optimize import DropRefcountingTransform
 149     from Optimize import ConsolidateOverflowCheck
 150     from Buffer import IntroduceBufferAuxiliaryVars
 151     from ModuleNode import check_c_declarations, check_c_declarations_pxd
 152
 153
 154     if mode == 'pxd':
 155         _check_c_declarations = check_c_declarations_pxd
 156         _specific_post_parse = PxdPostParse(context)
 157     else:
 158         _check_c_declarations = check_c_declarations
 159         _specific_post_parse = None
 160
 161     if mode == 'py':
 162         _align_function_definitions = AlignFunctionDefinitions(context)
 163     else:
 164         _align_function_definitions = None
 165
 166     # NOTE: This is the "common" parts of the pipeline, which is also
 167     # code in pxd files. So it will be run multiple times in a
 168     # compilation stage.
 169     stages = [
 170         NormalizeTree(context),
 171         PostParse(context),
 172         _specific_post_parse,
 173         InterpretCompilerDirectives(context, context.compiler_directives),
 174         ParallelRangeTransform(context),
 175         AdjustDefByDirectives(context),
 176         MarkClosureVisitor(context),
 177         _align_function_definitions,
 178         RemoveUnreachableCode(context),
 179         ConstantFolding(),
 180         FlattenInListTransform(),
 181         WithTransform(context),
 182         DecoratorTransform(context),
 183         ForwardDeclareTypes(context),
 184         AnalyseDeclarationsTransform(context),
 185         AutoTestDictTransform(context),
 186         EmbedSignature(context),
 187         EarlyReplaceBuiltinCalls(context),  ## Necessary?
 188         TransformBuiltinMethods(context),  ## Necessary?
 189         MarkParallelAssignments(context),
 190         ControlFlowAnalysis(context),
 191         RemoveUnreachableCode(context),
 192         # MarkParallelAssignments(context),
 193         MarkOverflowingArithmetic(context),
 194         IntroduceBufferAuxiliaryVars(context),
 195         _check_c_declarations,
 196         InlineDefNodeCalls(context),
 197         AnalyseExpressionsTransform(context),
 198         FindInvalidUseOfFusedTypes(context),
 199         ExpandInplaceOperators(context),
 200         OptimizeBuiltinCalls(context),  ## Necessary?
 201         CreateClosureClasses(context),  ## After all lookups and type inference
 202         CalculateQualifiedNamesTransform(context),
 203         ConsolidateOverflowCheck(context),
 204         IterationTransform(context),
 205         SwitchTransform(),
 206         DropRefcountingTransform(),
 207         FinalOptimizePhase(context),
 208         GilCheck(),
 209         UseUtilityCodeDefinitions(context),
 210         ]
 211     filtered_stages = []
 212     for s in stages:
 213         if s.__class__ not in exclude_classes:
 214             filtered_stages.append(s)
 215     return filtered_stages
 216
 217 def create_pyx_pipeline(context, options, result, py=False, exclude_classes=()):
 218     if py:
 219         mode = 'py'
 220     else:
 221         mode = 'pyx'
 222     test_support = []
 223     if options.evaluate_tree_assertions:
 224         from Cython.TestUtils import TreeAssertVisitor
 225         test_support.append(TreeAssertVisitor())
 226
 227     if options.gdb_debug:
 228         from Cython.Debugger import DebugWriter # requires Py2.5+
 229         from ParseTreeTransforms import DebugTransform
 230         context.gdb_debug_outputwriter = DebugWriter.CythonDebugWriter(
 231             options.output_dir)
 232         debug_transform = [DebugTransform(context, options, result)]
 233     else:
 234         debug_transform = []
 235
 236     return list(itertools.chain(
 237         [parse_stage_factory(context)],
 238         create_pipeline(context, mode, exclude_classes=exclude_classes),
 239         test_support,
 240         [inject_pxd_code_stage_factory(context),
 241          inject_utility_code_stage_factory(context),
 242          abort_on_errors],
 243         debug_transform,
 244         [generate_pyx_code_stage_factory(options, result)]))
 245
 246 def create_pxd_pipeline(context, scope, module_name):
 247     from CodeGeneration import ExtractPxdCode
 248
 249     # The pxd pipeline ends up with a CCodeWriter containing the
 250     # code of the pxd, as well as a pxd scope.
 251     return [
 252         parse_pxd_stage_factory(context, scope, module_name)
 253         ] + create_pipeline(context, 'pxd') + [
 254         ExtractPxdCode()
 255         ]
 256
 257 def create_py_pipeline(context, options, result):
 258     return create_pyx_pipeline(context, options, result, py=True)
 259
 260 def create_pyx_as_pxd_pipeline(context, result):
 261     from ParseTreeTransforms import AlignFunctionDefinitions, \
 262         MarkClosureVisitor, WithTransform, AnalyseDeclarationsTransform
 263     from Optimize import ConstantFolding, FlattenInListTransform
 264     from Nodes import StatListNode
 265     pipeline = []
 266     pyx_pipeline = create_pyx_pipeline(context, context.options, result,
 267                                        exclude_classes=[
 268                                            AlignFunctionDefinitions,
 269                                            MarkClosureVisitor,
 270                                            ConstantFolding,
 271                                            FlattenInListTransform,
 272                                            WithTransform
 273                                            ])
 274     for stage in pyx_pipeline:
 275         pipeline.append(stage)
 276         if isinstance(stage, AnalyseDeclarationsTransform):
 277             # This is the last stage we need.
 278             break
 279     def fake_pxd(root):
 280         for entry in root.scope.entries.values():
 281             if not entry.in_cinclude:
 282                 entry.defined_in_pxd = 1
 283                 if entry.name == entry.cname and entry.visibility != 'extern':
 284                     # Always mangle non-extern cimported entries.
 285                     entry.cname = entry.scope.mangle(Naming.func_prefix, entry.name)
 286         return StatListNode(root.pos, stats=[]), root.scope
 287     pipeline.append(fake_pxd)
 288     return pipeline
 289
 290 def insert_into_pipeline(pipeline, transform, before=None, after=None):
 291     """
 292     Insert a new transform into the pipeline after or before an instance of
 293     the given class. e.g.
 294
 295         pipeline = insert_into_pipeline(pipeline, transform,
 296                                         after=AnalyseDeclarationsTransform)
 297     """
 298     assert before or after
 299
 300     cls = before or after
 301     for i, t in enumerate(pipeline):
 302         if isinstance(t, cls):
 303             break
 304
 305     if after:
 306         i += 1
 307
 308     return pipeline[:i] + [transform] + pipeline[i:]
 309
 310 #
 311 # Running a pipeline
 312 #
 313
 314 def run_pipeline(pipeline, source, printtree=True):
 315     from Cython.Compiler.Visitor import PrintTree
 316
 317     error = None
 318     data = source
 319     try:
 320         try:
 321             for phase in pipeline:
 322                 if phase is not None:
 323                     if DebugFlags.debug_verbose_pipeline:
 324                         t = time()
 325                         print "Entering pipeline phase %r" % phase
 326                     if not printtree and isinstance(phase, PrintTree):
 327                         continue
 328                     data = phase(data)
 329                     if DebugFlags.debug_verbose_pipeline:
 330                         print "    %.3f seconds" % (time() - t)
 331         except CompileError, err:
 332             # err is set
 333             Errors.report_error(err)
 334             error = err
 335     except InternalError, err:
 336         # Only raise if there was not an earlier error
 337         if Errors.num_errors == 0:
 338             raise
 339         error = err
 340     except AbortError, err:
 341         error = err
 342     return (error, data)