5 import os
, sys
, re
, codecs
6 if sys
.version_info
[:2] < (2, 4):
7 sys
.stderr
.write("Sorry, Cython requires Python 2.4 or later\n")
11 # Do not import Parsing here, import it when needed, because Parsing imports
12 # Nodes, which globally needs debug command line options initialized to set a
13 # conditional metaclass. These options are processed by CmdLine called from
14 # main() in this file.
17 from Scanning
import PyrexScanner
, FileSourceDescriptor
18 from Errors
import PyrexError
, CompileError
, error
, warning
19 from Symtab
import ModuleScope
20 from Cython
import Utils
23 module_name_pattern
= re
.compile(r
"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$")
27 class CompilationData(object):
28 # Bundles the information that is passed from transform to transform.
29 # (For now, this is only)
31 # While Context contains every pxd ever loaded, path information etc.,
32 # this only contains the data related to a single compilation pass
34 # pyx ModuleNode Main code tree of this compilation.
35 # pxds {string : ModuleNode} Trees for the pxds used in the pyx.
36 # codewriter CCodeWriter Where to output final code.
37 # options CompilationOptions
38 # result CompilationResult
41 class Context(object):
42 # This class encapsulates the context needed for compiling
43 # one or more Cython implementation files along with their
44 # associated and imported declaration files. It includes
45 # the root of the module import namespace and the list
46 # of directories to search for include files.
48 # modules {string : ModuleScope}
49 # include_directories [string]
50 # future_directives [object]
51 # language_level int currently 2 or 3 for Python 2/3
55 def __init__(self
, include_directories
, compiler_directives
, cpp
=False,
56 language_level
=2, options
=None, create_testscope
=True):
57 # cython_scope is a hack, set to False by subclasses, in order to break
59 # Better code organization would fix it.
61 import Builtin
, CythonScope
62 self
.modules
= {"__builtin__" : Builtin
.builtin_scope
}
63 self
.cython_scope
= CythonScope
.create_cython_scope(self
)
64 self
.modules
["cython"] = self
.cython_scope
65 self
.include_directories
= include_directories
66 self
.future_directives
= set()
67 self
.compiler_directives
= compiler_directives
69 self
.options
= options
71 self
.pxds
= {} # full name -> node tree
73 standard_include_path
= os
.path
.abspath(os
.path
.normpath(
74 os
.path
.join(os
.path
.dirname(__file__
), os
.path
.pardir
, 'Includes')))
75 self
.include_directories
= include_directories
+ [standard_include_path
]
77 self
.set_language_level(language_level
)
79 self
.gdb_debug_outputwriter
= None
81 def set_language_level(self
, level
):
82 self
.language_level
= level
84 from Future
import print_function
, unicode_literals
, absolute_import
85 self
.future_directives
.update([print_function
, unicode_literals
, absolute_import
])
86 self
.modules
['builtins'] = self
.modules
['__builtin__']
88 # pipeline creation functions can now be found in Pipeline.py
90 def process_pxd(self
, source_desc
, scope
, module_name
):
92 if isinstance(source_desc
, FileSourceDescriptor
) and source_desc
._file
_type
== 'pyx':
93 source
= CompilationSource(source_desc
, module_name
, os
.getcwd())
94 result_sink
= create_default_resultobj(source
, self
.options
)
95 pipeline
= Pipeline
.create_pyx_as_pxd_pipeline(self
, result_sink
)
96 result
= Pipeline
.run_pipeline(pipeline
, source
)
98 pipeline
= Pipeline
.create_pxd_pipeline(self
, scope
, module_name
)
99 result
= Pipeline
.run_pipeline(pipeline
, source_desc
)
102 def nonfatal_error(self
, exc
):
103 return Errors
.report_error(exc
)
105 def find_module(self
, module_name
,
106 relative_to
= None, pos
= None, need_pxd
= 1, check_module_name
= True):
107 # Finds and returns the module scope corresponding to
108 # the given relative or absolute module name. If this
109 # is the first time the module has been requested, finds
110 # the corresponding .pxd file and process it.
111 # If relative_to is not None, it must be a module scope,
112 # and the module will first be searched for relative to
113 # that module, provided its name is not a dotted name.
114 debug_find_module
= 0
115 if debug_find_module
:
116 print("Context.find_module: module_name = %s, relative_to = %s, pos = %s, need_pxd = %s" % (
117 module_name
, relative_to
, pos
, need_pxd
))
121 if check_module_name
and not module_name_pattern
.match(module_name
):
123 pos
= (module_name
, 0, 0)
124 raise CompileError(pos
,
125 "'%s' is not a valid module name" % module_name
)
126 if "." not in module_name
and relative_to
:
127 if debug_find_module
:
128 print("...trying relative import")
129 scope
= relative_to
.lookup_submodule(module_name
)
131 qualified_name
= relative_to
.qualify_name(module_name
)
132 pxd_pathname
= self
.find_pxd_file(qualified_name
, pos
)
134 scope
= relative_to
.find_submodule(module_name
)
136 if debug_find_module
:
137 print("...trying absolute import")
139 for name
in module_name
.split("."):
140 scope
= scope
.find_submodule(name
)
141 if debug_find_module
:
142 print("...scope =", scope
)
143 if not scope
.pxd_file_loaded
:
144 if debug_find_module
:
145 print("...pxd not loaded")
146 scope
.pxd_file_loaded
= 1
148 if debug_find_module
:
149 print("...looking for pxd file")
150 pxd_pathname
= self
.find_pxd_file(module_name
, pos
)
151 if debug_find_module
:
152 print("......found ", pxd_pathname
)
153 if not pxd_pathname
and need_pxd
:
154 package_pathname
= self
.search_include_directories(module_name
, ".py", pos
)
155 if package_pathname
and package_pathname
.endswith('__init__.py'):
158 error(pos
, "'%s.pxd' not found" % module_name
)
161 if debug_find_module
:
162 print("Context.find_module: Parsing %s" % pxd_pathname
)
163 rel_path
= module_name
.replace('.', os
.sep
) + os
.path
.splitext(pxd_pathname
)[1]
164 if not pxd_pathname
.endswith(rel_path
):
165 rel_path
= pxd_pathname
# safety measure to prevent printing incorrect paths
166 source_desc
= FileSourceDescriptor(pxd_pathname
, rel_path
)
167 err
, result
= self
.process_pxd(source_desc
, scope
, module_name
)
170 (pxd_codenodes
, pxd_scope
) = result
171 self
.pxds
[module_name
] = (pxd_codenodes
, pxd_scope
)
176 def find_pxd_file(self
, qualified_name
, pos
):
177 # Search include path for the .pxd file corresponding to the
178 # given fully-qualified module name.
179 # Will find either a dotted filename or a file in a
180 # package directory. If a source file position is given,
181 # the directory containing the source file is searched first
182 # for a dotted filename, and its containing package root
183 # directory is searched first for a non-dotted filename.
184 pxd
= self
.search_include_directories(qualified_name
, ".pxd", pos
, sys_path
=True)
185 if pxd
is None: # XXX Keep this until Includes/Deprecated is removed
186 if (qualified_name
.startswith('python') or
187 qualified_name
in ('stdlib', 'stdio', 'stl')):
188 standard_include_path
= os
.path
.abspath(os
.path
.normpath(
189 os
.path
.join(os
.path
.dirname(__file__
), os
.path
.pardir
, 'Includes')))
190 deprecated_include_path
= os
.path
.join(standard_include_path
, 'Deprecated')
191 self
.include_directories
.append(deprecated_include_path
)
193 pxd
= self
.search_include_directories(qualified_name
, ".pxd", pos
)
195 self
.include_directories
.pop()
197 name
= qualified_name
198 if name
.startswith('python'):
199 warning(pos
, "'%s' is deprecated, use 'cpython'" % name
, 1)
200 elif name
in ('stdlib', 'stdio'):
201 warning(pos
, "'%s' is deprecated, use 'libc.%s'" % (name
, name
), 1)
202 elif name
in ('stl'):
203 warning(pos
, "'%s' is deprecated, use 'libcpp.*.*'" % name
, 1)
204 if pxd
is None and Options
.cimport_from_pyx
:
205 return self
.find_pyx_file(qualified_name
, pos
)
208 def find_pyx_file(self
, qualified_name
, pos
):
209 # Search include path for the .pyx file corresponding to the
210 # given fully-qualified module name, as for find_pxd_file().
211 return self
.search_include_directories(qualified_name
, ".pyx", pos
)
213 def find_include_file(self
, filename
, pos
):
214 # Search list of include directories for filename.
215 # Reports an error and returns None if not found.
216 path
= self
.search_include_directories(filename
, "", pos
,
219 error(pos
, "'%s' not found" % filename
)
222 def search_include_directories(self
, qualified_name
, suffix
, pos
,
223 include
=False, sys_path
=False):
224 return Utils
.search_include_directories(
225 tuple(self
.include_directories
), qualified_name
, suffix
, pos
, include
, sys_path
)
227 def find_root_package_dir(self
, file_path
):
228 return Utils
.find_root_package_dir(file_path
)
230 def check_package_dir(self
, dir, package_names
):
231 return Utils
.check_package_dir(dir, tuple(package_names
))
233 def c_file_out_of_date(self
, source_path
):
234 c_path
= Utils
.replace_suffix(source_path
, ".c")
235 if not os
.path
.exists(c_path
):
237 c_time
= Utils
.modification_time(c_path
)
238 if Utils
.file_newer_than(source_path
, c_time
):
241 pxd_path
= Utils
.replace_suffix(source_path
, ".pxd")
242 if os
.path
.exists(pxd_path
) and Utils
.file_newer_than(pxd_path
, c_time
):
244 for kind
, name
in self
.read_dependency_file(source_path
):
245 if kind
== "cimport":
246 dep_path
= self
.find_pxd_file(name
, pos
)
247 elif kind
== "include":
248 dep_path
= self
.search_include_directories(name
, pos
)
251 if dep_path
and Utils
.file_newer_than(dep_path
, c_time
):
255 def find_cimported_module_names(self
, source_path
):
256 return [ name
for kind
, name
in self
.read_dependency_file(source_path
)
257 if kind
== "cimport" ]
259 def is_package_dir(self
, dir_path
):
260 return Utils
.is_package_dir(dir_path
)
262 def read_dependency_file(self
, source_path
):
263 dep_path
= Utils
.replace_suffix(source_path
, ".dep")
264 if os
.path
.exists(dep_path
):
265 f
= open(dep_path
, "rU")
266 chunks
= [ line
.strip().split(" ", 1)
267 for line
in f
.readlines()
268 if " " in line
.strip() ]
274 def lookup_submodule(self
, name
):
275 # Look up a top-level module. Returns None if not found.
276 return self
.modules
.get(name
, None)
278 def find_submodule(self
, name
):
279 # Find a top-level module, creating a new one if needed.
280 scope
= self
.lookup_submodule(name
)
282 scope
= ModuleScope(name
,
283 parent_module
= None, context
= self
)
284 self
.modules
[name
] = scope
287 def parse(self
, source_desc
, scope
, pxd
, full_module_name
):
288 if not isinstance(source_desc
, FileSourceDescriptor
):
289 raise RuntimeError("Only file sources for code supported")
290 source_filename
= source_desc
.filename
292 # Parse the given source file and return a parse tree.
293 num_errors
= Errors
.num_errors
295 f
= Utils
.open_source_file(source_filename
, "rU")
298 s
= PyrexScanner(f
, source_desc
, source_encoding
= f
.encoding
,
299 scope
= scope
, context
= self
)
300 tree
= Parsing
.p_module(s
, pxd
, full_module_name
)
303 except UnicodeDecodeError, e
:
305 #traceback.print_exc()
312 f
= open(source_filename
, "rb")
318 # FIXME: make this at least a little less inefficient
319 for idx
, c
in enumerate(byte_data
):
320 if c
in (ord('\n'), '\n'):
328 error((source_desc
, line
, column
),
329 "Decoding error, missing or incorrect coding=<encoding-name> "
330 "at top of source (cannot decode with encoding %r: %s)" % (encoding
, msg
))
332 if Errors
.num_errors
> num_errors
:
336 def extract_module_name(self
, path
, options
):
337 # Find fully_qualified module name from the full pathname
339 dir, filename
= os
.path
.split(path
)
340 module_name
, _
= os
.path
.splitext(filename
)
341 if "." in module_name
:
343 names
= [module_name
]
344 while self
.is_package_dir(dir):
345 parent
, package_name
= os
.path
.split(dir)
348 names
.append(package_name
)
351 return ".".join(names
)
353 def setup_errors(self
, options
, result
):
354 Errors
.reset() # clear any remaining error state
355 if options
.use_listing_file
:
356 result
.listing_file
= Utils
.replace_suffix(source
, ".lis")
357 path
= result
.listing_file
360 Errors
.open_listing_file(path
=path
,
361 echo_to_stderr
=options
.errors_to_stderr
)
363 def teardown_errors(self
, err
, options
, result
):
364 source_desc
= result
.compilation_source
.source_desc
365 if not isinstance(source_desc
, FileSourceDescriptor
):
366 raise RuntimeError("Only file sources for code supported")
367 Errors
.close_listing_file()
368 result
.num_errors
= Errors
.num_errors
369 if result
.num_errors
> 0:
371 if err
and result
.c_file
:
373 Utils
.castrate_file(result
.c_file
, os
.stat(source_desc
.filename
))
374 except EnvironmentError:
378 def create_default_resultobj(compilation_source
, options
):
379 result
= CompilationResult()
380 result
.main_source_file
= compilation_source
.source_desc
.filename
381 result
.compilation_source
= compilation_source
382 source_desc
= compilation_source
.source_desc
383 if options
.output_file
:
384 result
.c_file
= os
.path
.join(compilation_source
.cwd
, options
.output_file
)
390 result
.c_file
= Utils
.replace_suffix(source_desc
.filename
, c_suffix
)
393 def run_pipeline(source
, options
, full_module_name
=None, context
=None):
396 source_ext
= os
.path
.splitext(source
)[1]
397 options
.configure_language_defaults(source_ext
[1:]) # py/pyx
399 context
= options
.create_context()
401 # Set up source object
403 abs_path
= os
.path
.abspath(source
)
404 full_module_name
= full_module_name
or context
.extract_module_name(source
, options
)
406 if options
.relative_path_in_code_position_comments
:
407 rel_path
= full_module_name
.replace('.', os
.sep
) + source_ext
408 if not abs_path
.endswith(rel_path
):
409 rel_path
= source
# safety measure to prevent printing incorrect paths
412 source_desc
= FileSourceDescriptor(abs_path
, rel_path
)
413 source
= CompilationSource(source_desc
, full_module_name
, cwd
)
415 # Set up result object
416 result
= create_default_resultobj(source
, options
)
418 if options
.annotate
is None:
419 # By default, decide based on whether an html file already exists.
420 html_filename
= os
.path
.splitext(result
.c_file
)[0] + ".html"
421 if os
.path
.exists(html_filename
):
422 line
= codecs
.open(html_filename
, "r", encoding
="UTF-8").readline()
423 if line
.startswith(u
'<!-- Generated by Cython'):
424 options
.annotate
= True
427 if source_ext
.lower() == '.py' or not source_ext
:
428 pipeline
= Pipeline
.create_py_pipeline(context
, options
, result
)
430 pipeline
= Pipeline
.create_pyx_pipeline(context
, options
, result
)
432 context
.setup_errors(options
, result
)
433 err
, enddata
= Pipeline
.run_pipeline(pipeline
, source
)
434 context
.teardown_errors(err
, options
, result
)
438 #------------------------------------------------------------------------
440 # Main Python entry points
442 #------------------------------------------------------------------------
444 class CompilationSource(object):
446 Contains the data necesarry to start up a compilation pipeline for
447 a single compilation unit.
449 def __init__(self
, source_desc
, full_module_name
, cwd
):
450 self
.source_desc
= source_desc
451 self
.full_module_name
= full_module_name
454 class CompilationOptions(object):
456 Options to the Cython compiler:
458 show_version boolean Display version number
459 use_listing_file boolean Generate a .lis file
460 errors_to_stderr boolean Echo errors to stderr when using .lis
461 include_path [string] Directories to search for include files
462 output_file string Name of generated .c file
463 generate_pxi boolean Generate .pxi file for public declarations
464 capi_reexport_cincludes
465 boolean Add cincluded headers to any auto-generated
467 timestamps boolean Only compile changed source files.
468 verbose boolean Always print source names being compiled
469 compiler_directives dict Overrides for pragma options (see Options.py)
470 evaluate_tree_assertions boolean Test support: evaluate parse tree assertions
471 language_level integer The Python language level: 2 or 3
473 cplus boolean Compile as c++ code
476 def __init__(self
, defaults
= None, **kw
):
477 self
.include_path
= []
479 if isinstance(defaults
, CompilationOptions
):
480 defaults
= defaults
.__dict
__
482 defaults
= default_options
484 options
= dict(defaults
)
487 directives
= dict(options
['compiler_directives']) # copy mutable field
488 options
['compiler_directives'] = directives
489 if 'language_level' in directives
and 'language_level' not in kw
:
490 options
['language_level'] = int(directives
['language_level'])
491 if 'cache' in options
:
492 if options
['cache'] is True:
493 options
['cache'] = os
.path
.expanduser("~/.cycache")
494 elif options
['cache'] in (False, None):
497 self
.__dict
__.update(options
)
499 def configure_language_defaults(self
, source_extension
):
500 if source_extension
== 'py':
501 if self
.compiler_directives
.get('binding') is None:
502 self
.compiler_directives
['binding'] = True
504 def create_context(self
):
505 return Context(self
.include_path
, self
.compiler_directives
,
506 self
.cplus
, self
.language_level
, options
=self
)
509 class CompilationResult(object):
511 Results from the Cython compiler:
513 c_file string or None The generated C source file
514 h_file string or None The generated C header file
515 i_file string or None The generated .pxi file
516 api_file string or None The generated C API .h file
517 listing_file string or None File of error messages
518 object_file string or None Result of compiling the C file
519 extension_file string or None Result of linking the object file
520 num_errors integer Number of compilation errors
521 compilation_source CompilationSource
529 self
.listing_file
= None
530 self
.object_file
= None
531 self
.extension_file
= None
532 self
.main_source_file
= None
535 class CompilationResultSet(dict):
537 Results from compiling multiple Pyrex source files. A mapping
538 from source file paths to CompilationResult instances. Also
539 has the following attributes:
541 num_errors integer Total number of compilation errors
546 def add(self
, source
, result
):
547 self
[source
] = result
548 self
.num_errors
+= result
.num_errors
551 def compile_single(source
, options
, full_module_name
= None):
553 compile_single(source, options, full_module_name)
555 Compile the given Pyrex implementation file and return a CompilationResult.
556 Always compiles a single file; does not perform timestamp checking or
559 return run_pipeline(source
, options
, full_module_name
)
562 def compile_multiple(sources
, options
):
564 compile_multiple(sources, options)
566 Compiles the given sequence of Pyrex implementation files and returns
567 a CompilationResultSet. Performs timestamp checking and/or recursion
568 if these are specified in the options.
570 # run_pipeline creates the context
571 # context = options.create_context()
572 sources
= [os
.path
.abspath(source
) for source
in sources
]
574 results
= CompilationResultSet()
575 timestamps
= options
.timestamps
576 verbose
= options
.verbose
578 for source
in sources
:
579 if source
not in processed
:
581 context
= options
.create_context()
582 if not timestamps
or context
.c_file_out_of_date(source
):
584 sys
.stderr
.write("Compiling %s\n" % source
)
586 result
= run_pipeline(source
, options
, context
=context
)
587 results
.add(source
, result
)
588 # Compiling multiple sources in one context doesn't quite
591 processed
.add(source
)
594 def compile(source
, options
= None, full_module_name
= None, **kwds
):
596 compile(source [, options], [, <option> = <value>]...)
598 Compile one or more Pyrex implementation files, with optional timestamp
599 checking and recursing on dependecies. The source argument may be a string
600 or a sequence of strings If it is a string and no recursion or timestamp
601 checking is requested, a CompilationResult is returned, otherwise a
602 CompilationResultSet is returned.
604 options
= CompilationOptions(defaults
= options
, **kwds
)
605 if isinstance(source
, basestring
) and not options
.timestamps
:
606 return compile_single(source
, options
, full_module_name
)
608 return compile_multiple(source
, options
)
610 #------------------------------------------------------------------------
612 # Main command-line entry point
614 #------------------------------------------------------------------------
615 def setuptools_main():
616 return main(command_line
= 1)
618 def main(command_line
= 0):
622 from CmdLine
import parse_command_line
623 options
, sources
= parse_command_line(args
)
625 options
= CompilationOptions(default_options
)
628 if options
.show_version
:
629 sys
.stderr
.write("Cython version %s\n" % Version
.version
)
630 if options
.working_path
!="":
631 os
.chdir(options
.working_path
)
633 result
= compile(sources
, options
)
634 if result
.num_errors
> 0:
636 except (EnvironmentError, PyrexError
), e
:
637 sys
.stderr
.write(str(e
) + '\n')
644 #------------------------------------------------------------------------
646 # Set the default options depending on the platform
648 #------------------------------------------------------------------------
650 default_options
= dict(
652 use_listing_file
= 0,
653 errors_to_stderr
= 1,
658 capi_reexport_cincludes
= 0,
663 compiler_directives
= {},
664 evaluate_tree_assertions
= False,
665 emit_linenums
= False,
666 relative_path_in_code_position_comments
= True,
667 c_line_in_traceback
= True,
670 compile_time_env
= None,
671 common_utility_include_dir
= None,