Pin Chrome's shortcut to the Win10 Start menu on install and OS upgrade.
[chromium-blink-merge.git] / third_party / cython / src / Cython / Compiler / Main.py
blob727fe9bb521afa76df089405b2a159c3e733cd85
2 # Cython Top Level
5 import os, sys, re, codecs
6 if sys.version_info[:2] < (2, 4):
7 sys.stderr.write("Sorry, Cython requires Python 2.4 or later\n")
8 sys.exit(1)
10 import Errors
11 # Do not import Parsing here, import it when needed, because Parsing imports
12 # Nodes, which globally needs debug command line options initialized to set a
13 # conditional metaclass. These options are processed by CmdLine called from
14 # main() in this file.
15 # import Parsing
16 import Version
17 from Scanning import PyrexScanner, FileSourceDescriptor
18 from Errors import PyrexError, CompileError, error, warning
19 from Symtab import ModuleScope
20 from Cython import Utils
21 import Options
23 module_name_pattern = re.compile(r"[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)*$")
25 verbose = 0
27 class CompilationData(object):
28 # Bundles the information that is passed from transform to transform.
29 # (For now, this is only)
31 # While Context contains every pxd ever loaded, path information etc.,
32 # this only contains the data related to a single compilation pass
34 # pyx ModuleNode Main code tree of this compilation.
35 # pxds {string : ModuleNode} Trees for the pxds used in the pyx.
36 # codewriter CCodeWriter Where to output final code.
37 # options CompilationOptions
38 # result CompilationResult
39 pass
41 class Context(object):
42 # This class encapsulates the context needed for compiling
43 # one or more Cython implementation files along with their
44 # associated and imported declaration files. It includes
45 # the root of the module import namespace and the list
46 # of directories to search for include files.
48 # modules {string : ModuleScope}
49 # include_directories [string]
50 # future_directives [object]
51 # language_level int currently 2 or 3 for Python 2/3
53 cython_scope = None
55 def __init__(self, include_directories, compiler_directives, cpp=False,
56 language_level=2, options=None, create_testscope=True):
57 # cython_scope is a hack, set to False by subclasses, in order to break
58 # an infinite loop.
59 # Better code organization would fix it.
61 import Builtin, CythonScope
62 self.modules = {"__builtin__" : Builtin.builtin_scope}
63 self.cython_scope = CythonScope.create_cython_scope(self)
64 self.modules["cython"] = self.cython_scope
65 self.include_directories = include_directories
66 self.future_directives = set()
67 self.compiler_directives = compiler_directives
68 self.cpp = cpp
69 self.options = options
71 self.pxds = {} # full name -> node tree
73 standard_include_path = os.path.abspath(os.path.normpath(
74 os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes')))
75 self.include_directories = include_directories + [standard_include_path]
77 self.set_language_level(language_level)
79 self.gdb_debug_outputwriter = None
81 def set_language_level(self, level):
82 self.language_level = level
83 if level >= 3:
84 from Future import print_function, unicode_literals, absolute_import
85 self.future_directives.update([print_function, unicode_literals, absolute_import])
86 self.modules['builtins'] = self.modules['__builtin__']
88 # pipeline creation functions can now be found in Pipeline.py
90 def process_pxd(self, source_desc, scope, module_name):
91 import Pipeline
92 if isinstance(source_desc, FileSourceDescriptor) and source_desc._file_type == 'pyx':
93 source = CompilationSource(source_desc, module_name, os.getcwd())
94 result_sink = create_default_resultobj(source, self.options)
95 pipeline = Pipeline.create_pyx_as_pxd_pipeline(self, result_sink)
96 result = Pipeline.run_pipeline(pipeline, source)
97 else:
98 pipeline = Pipeline.create_pxd_pipeline(self, scope, module_name)
99 result = Pipeline.run_pipeline(pipeline, source_desc)
100 return result
102 def nonfatal_error(self, exc):
103 return Errors.report_error(exc)
105 def find_module(self, module_name,
106 relative_to = None, pos = None, need_pxd = 1, check_module_name = True):
107 # Finds and returns the module scope corresponding to
108 # the given relative or absolute module name. If this
109 # is the first time the module has been requested, finds
110 # the corresponding .pxd file and process it.
111 # If relative_to is not None, it must be a module scope,
112 # and the module will first be searched for relative to
113 # that module, provided its name is not a dotted name.
114 debug_find_module = 0
115 if debug_find_module:
116 print("Context.find_module: module_name = %s, relative_to = %s, pos = %s, need_pxd = %s" % (
117 module_name, relative_to, pos, need_pxd))
119 scope = None
120 pxd_pathname = None
121 if check_module_name and not module_name_pattern.match(module_name):
122 if pos is None:
123 pos = (module_name, 0, 0)
124 raise CompileError(pos,
125 "'%s' is not a valid module name" % module_name)
126 if "." not in module_name and relative_to:
127 if debug_find_module:
128 print("...trying relative import")
129 scope = relative_to.lookup_submodule(module_name)
130 if not scope:
131 qualified_name = relative_to.qualify_name(module_name)
132 pxd_pathname = self.find_pxd_file(qualified_name, pos)
133 if pxd_pathname:
134 scope = relative_to.find_submodule(module_name)
135 if not scope:
136 if debug_find_module:
137 print("...trying absolute import")
138 scope = self
139 for name in module_name.split("."):
140 scope = scope.find_submodule(name)
141 if debug_find_module:
142 print("...scope =", scope)
143 if not scope.pxd_file_loaded:
144 if debug_find_module:
145 print("...pxd not loaded")
146 scope.pxd_file_loaded = 1
147 if not pxd_pathname:
148 if debug_find_module:
149 print("...looking for pxd file")
150 pxd_pathname = self.find_pxd_file(module_name, pos)
151 if debug_find_module:
152 print("......found ", pxd_pathname)
153 if not pxd_pathname and need_pxd:
154 package_pathname = self.search_include_directories(module_name, ".py", pos)
155 if package_pathname and package_pathname.endswith('__init__.py'):
156 pass
157 else:
158 error(pos, "'%s.pxd' not found" % module_name)
159 if pxd_pathname:
160 try:
161 if debug_find_module:
162 print("Context.find_module: Parsing %s" % pxd_pathname)
163 rel_path = module_name.replace('.', os.sep) + os.path.splitext(pxd_pathname)[1]
164 if not pxd_pathname.endswith(rel_path):
165 rel_path = pxd_pathname # safety measure to prevent printing incorrect paths
166 source_desc = FileSourceDescriptor(pxd_pathname, rel_path)
167 err, result = self.process_pxd(source_desc, scope, module_name)
168 if err:
169 raise err
170 (pxd_codenodes, pxd_scope) = result
171 self.pxds[module_name] = (pxd_codenodes, pxd_scope)
172 except CompileError:
173 pass
174 return scope
176 def find_pxd_file(self, qualified_name, pos):
177 # Search include path for the .pxd file corresponding to the
178 # given fully-qualified module name.
179 # Will find either a dotted filename or a file in a
180 # package directory. If a source file position is given,
181 # the directory containing the source file is searched first
182 # for a dotted filename, and its containing package root
183 # directory is searched first for a non-dotted filename.
184 pxd = self.search_include_directories(qualified_name, ".pxd", pos, sys_path=True)
185 if pxd is None: # XXX Keep this until Includes/Deprecated is removed
186 if (qualified_name.startswith('python') or
187 qualified_name in ('stdlib', 'stdio', 'stl')):
188 standard_include_path = os.path.abspath(os.path.normpath(
189 os.path.join(os.path.dirname(__file__), os.path.pardir, 'Includes')))
190 deprecated_include_path = os.path.join(standard_include_path, 'Deprecated')
191 self.include_directories.append(deprecated_include_path)
192 try:
193 pxd = self.search_include_directories(qualified_name, ".pxd", pos)
194 finally:
195 self.include_directories.pop()
196 if pxd:
197 name = qualified_name
198 if name.startswith('python'):
199 warning(pos, "'%s' is deprecated, use 'cpython'" % name, 1)
200 elif name in ('stdlib', 'stdio'):
201 warning(pos, "'%s' is deprecated, use 'libc.%s'" % (name, name), 1)
202 elif name in ('stl'):
203 warning(pos, "'%s' is deprecated, use 'libcpp.*.*'" % name, 1)
204 if pxd is None and Options.cimport_from_pyx:
205 return self.find_pyx_file(qualified_name, pos)
206 return pxd
208 def find_pyx_file(self, qualified_name, pos):
209 # Search include path for the .pyx file corresponding to the
210 # given fully-qualified module name, as for find_pxd_file().
211 return self.search_include_directories(qualified_name, ".pyx", pos)
213 def find_include_file(self, filename, pos):
214 # Search list of include directories for filename.
215 # Reports an error and returns None if not found.
216 path = self.search_include_directories(filename, "", pos,
217 include=True)
218 if not path:
219 error(pos, "'%s' not found" % filename)
220 return path
222 def search_include_directories(self, qualified_name, suffix, pos,
223 include=False, sys_path=False):
224 return Utils.search_include_directories(
225 tuple(self.include_directories), qualified_name, suffix, pos, include, sys_path)
227 def find_root_package_dir(self, file_path):
228 return Utils.find_root_package_dir(file_path)
230 def check_package_dir(self, dir, package_names):
231 return Utils.check_package_dir(dir, tuple(package_names))
233 def c_file_out_of_date(self, source_path):
234 c_path = Utils.replace_suffix(source_path, ".c")
235 if not os.path.exists(c_path):
236 return 1
237 c_time = Utils.modification_time(c_path)
238 if Utils.file_newer_than(source_path, c_time):
239 return 1
240 pos = [source_path]
241 pxd_path = Utils.replace_suffix(source_path, ".pxd")
242 if os.path.exists(pxd_path) and Utils.file_newer_than(pxd_path, c_time):
243 return 1
244 for kind, name in self.read_dependency_file(source_path):
245 if kind == "cimport":
246 dep_path = self.find_pxd_file(name, pos)
247 elif kind == "include":
248 dep_path = self.search_include_directories(name, pos)
249 else:
250 continue
251 if dep_path and Utils.file_newer_than(dep_path, c_time):
252 return 1
253 return 0
255 def find_cimported_module_names(self, source_path):
256 return [ name for kind, name in self.read_dependency_file(source_path)
257 if kind == "cimport" ]
259 def is_package_dir(self, dir_path):
260 return Utils.is_package_dir(dir_path)
262 def read_dependency_file(self, source_path):
263 dep_path = Utils.replace_suffix(source_path, ".dep")
264 if os.path.exists(dep_path):
265 f = open(dep_path, "rU")
266 chunks = [ line.strip().split(" ", 1)
267 for line in f.readlines()
268 if " " in line.strip() ]
269 f.close()
270 return chunks
271 else:
272 return ()
274 def lookup_submodule(self, name):
275 # Look up a top-level module. Returns None if not found.
276 return self.modules.get(name, None)
278 def find_submodule(self, name):
279 # Find a top-level module, creating a new one if needed.
280 scope = self.lookup_submodule(name)
281 if not scope:
282 scope = ModuleScope(name,
283 parent_module = None, context = self)
284 self.modules[name] = scope
285 return scope
287 def parse(self, source_desc, scope, pxd, full_module_name):
288 if not isinstance(source_desc, FileSourceDescriptor):
289 raise RuntimeError("Only file sources for code supported")
290 source_filename = source_desc.filename
291 scope.cpp = self.cpp
292 # Parse the given source file and return a parse tree.
293 num_errors = Errors.num_errors
294 try:
295 f = Utils.open_source_file(source_filename, "rU")
296 try:
297 import Parsing
298 s = PyrexScanner(f, source_desc, source_encoding = f.encoding,
299 scope = scope, context = self)
300 tree = Parsing.p_module(s, pxd, full_module_name)
301 finally:
302 f.close()
303 except UnicodeDecodeError, e:
304 #import traceback
305 #traceback.print_exc()
306 line = 1
307 column = 0
308 msg = e.args[-1]
309 position = e.args[2]
310 encoding = e.args[0]
312 f = open(source_filename, "rb")
313 try:
314 byte_data = f.read()
315 finally:
316 f.close()
318 # FIXME: make this at least a little less inefficient
319 for idx, c in enumerate(byte_data):
320 if c in (ord('\n'), '\n'):
321 line += 1
322 column = 0
323 if idx == position:
324 break
326 column += 1
328 error((source_desc, line, column),
329 "Decoding error, missing or incorrect coding=<encoding-name> "
330 "at top of source (cannot decode with encoding %r: %s)" % (encoding, msg))
332 if Errors.num_errors > num_errors:
333 raise CompileError()
334 return tree
336 def extract_module_name(self, path, options):
337 # Find fully_qualified module name from the full pathname
338 # of a source file.
339 dir, filename = os.path.split(path)
340 module_name, _ = os.path.splitext(filename)
341 if "." in module_name:
342 return module_name
343 names = [module_name]
344 while self.is_package_dir(dir):
345 parent, package_name = os.path.split(dir)
346 if parent == dir:
347 break
348 names.append(package_name)
349 dir = parent
350 names.reverse()
351 return ".".join(names)
353 def setup_errors(self, options, result):
354 Errors.reset() # clear any remaining error state
355 if options.use_listing_file:
356 result.listing_file = Utils.replace_suffix(source, ".lis")
357 path = result.listing_file
358 else:
359 path = None
360 Errors.open_listing_file(path=path,
361 echo_to_stderr=options.errors_to_stderr)
363 def teardown_errors(self, err, options, result):
364 source_desc = result.compilation_source.source_desc
365 if not isinstance(source_desc, FileSourceDescriptor):
366 raise RuntimeError("Only file sources for code supported")
367 Errors.close_listing_file()
368 result.num_errors = Errors.num_errors
369 if result.num_errors > 0:
370 err = True
371 if err and result.c_file:
372 try:
373 Utils.castrate_file(result.c_file, os.stat(source_desc.filename))
374 except EnvironmentError:
375 pass
376 result.c_file = None
378 def create_default_resultobj(compilation_source, options):
379 result = CompilationResult()
380 result.main_source_file = compilation_source.source_desc.filename
381 result.compilation_source = compilation_source
382 source_desc = compilation_source.source_desc
383 if options.output_file:
384 result.c_file = os.path.join(compilation_source.cwd, options.output_file)
385 else:
386 if options.cplus:
387 c_suffix = ".cpp"
388 else:
389 c_suffix = ".c"
390 result.c_file = Utils.replace_suffix(source_desc.filename, c_suffix)
391 return result
393 def run_pipeline(source, options, full_module_name=None, context=None):
394 import Pipeline
396 source_ext = os.path.splitext(source)[1]
397 options.configure_language_defaults(source_ext[1:]) # py/pyx
398 if context is None:
399 context = options.create_context()
401 # Set up source object
402 cwd = os.getcwd()
403 abs_path = os.path.abspath(source)
404 full_module_name = full_module_name or context.extract_module_name(source, options)
406 if options.relative_path_in_code_position_comments:
407 rel_path = full_module_name.replace('.', os.sep) + source_ext
408 if not abs_path.endswith(rel_path):
409 rel_path = source # safety measure to prevent printing incorrect paths
410 else:
411 rel_path = abs_path
412 source_desc = FileSourceDescriptor(abs_path, rel_path)
413 source = CompilationSource(source_desc, full_module_name, cwd)
415 # Set up result object
416 result = create_default_resultobj(source, options)
418 if options.annotate is None:
419 # By default, decide based on whether an html file already exists.
420 html_filename = os.path.splitext(result.c_file)[0] + ".html"
421 if os.path.exists(html_filename):
422 line = codecs.open(html_filename, "r", encoding="UTF-8").readline()
423 if line.startswith(u'<!-- Generated by Cython'):
424 options.annotate = True
426 # Get pipeline
427 if source_ext.lower() == '.py' or not source_ext:
428 pipeline = Pipeline.create_py_pipeline(context, options, result)
429 else:
430 pipeline = Pipeline.create_pyx_pipeline(context, options, result)
432 context.setup_errors(options, result)
433 err, enddata = Pipeline.run_pipeline(pipeline, source)
434 context.teardown_errors(err, options, result)
435 return result
438 #------------------------------------------------------------------------
440 # Main Python entry points
442 #------------------------------------------------------------------------
444 class CompilationSource(object):
446 Contains the data necesarry to start up a compilation pipeline for
447 a single compilation unit.
449 def __init__(self, source_desc, full_module_name, cwd):
450 self.source_desc = source_desc
451 self.full_module_name = full_module_name
452 self.cwd = cwd
454 class CompilationOptions(object):
456 Options to the Cython compiler:
458 show_version boolean Display version number
459 use_listing_file boolean Generate a .lis file
460 errors_to_stderr boolean Echo errors to stderr when using .lis
461 include_path [string] Directories to search for include files
462 output_file string Name of generated .c file
463 generate_pxi boolean Generate .pxi file for public declarations
464 capi_reexport_cincludes
465 boolean Add cincluded headers to any auto-generated
466 header files.
467 timestamps boolean Only compile changed source files.
468 verbose boolean Always print source names being compiled
469 compiler_directives dict Overrides for pragma options (see Options.py)
470 evaluate_tree_assertions boolean Test support: evaluate parse tree assertions
471 language_level integer The Python language level: 2 or 3
473 cplus boolean Compile as c++ code
476 def __init__(self, defaults = None, **kw):
477 self.include_path = []
478 if defaults:
479 if isinstance(defaults, CompilationOptions):
480 defaults = defaults.__dict__
481 else:
482 defaults = default_options
484 options = dict(defaults)
485 options.update(kw)
487 directives = dict(options['compiler_directives']) # copy mutable field
488 options['compiler_directives'] = directives
489 if 'language_level' in directives and 'language_level' not in kw:
490 options['language_level'] = int(directives['language_level'])
491 if 'cache' in options:
492 if options['cache'] is True:
493 options['cache'] = os.path.expanduser("~/.cycache")
494 elif options['cache'] in (False, None):
495 del options['cache']
497 self.__dict__.update(options)
499 def configure_language_defaults(self, source_extension):
500 if source_extension == 'py':
501 if self.compiler_directives.get('binding') is None:
502 self.compiler_directives['binding'] = True
504 def create_context(self):
505 return Context(self.include_path, self.compiler_directives,
506 self.cplus, self.language_level, options=self)
509 class CompilationResult(object):
511 Results from the Cython compiler:
513 c_file string or None The generated C source file
514 h_file string or None The generated C header file
515 i_file string or None The generated .pxi file
516 api_file string or None The generated C API .h file
517 listing_file string or None File of error messages
518 object_file string or None Result of compiling the C file
519 extension_file string or None Result of linking the object file
520 num_errors integer Number of compilation errors
521 compilation_source CompilationSource
524 def __init__(self):
525 self.c_file = None
526 self.h_file = None
527 self.i_file = None
528 self.api_file = None
529 self.listing_file = None
530 self.object_file = None
531 self.extension_file = None
532 self.main_source_file = None
535 class CompilationResultSet(dict):
537 Results from compiling multiple Pyrex source files. A mapping
538 from source file paths to CompilationResult instances. Also
539 has the following attributes:
541 num_errors integer Total number of compilation errors
544 num_errors = 0
546 def add(self, source, result):
547 self[source] = result
548 self.num_errors += result.num_errors
551 def compile_single(source, options, full_module_name = None):
553 compile_single(source, options, full_module_name)
555 Compile the given Pyrex implementation file and return a CompilationResult.
556 Always compiles a single file; does not perform timestamp checking or
557 recursion.
559 return run_pipeline(source, options, full_module_name)
562 def compile_multiple(sources, options):
564 compile_multiple(sources, options)
566 Compiles the given sequence of Pyrex implementation files and returns
567 a CompilationResultSet. Performs timestamp checking and/or recursion
568 if these are specified in the options.
570 # run_pipeline creates the context
571 # context = options.create_context()
572 sources = [os.path.abspath(source) for source in sources]
573 processed = set()
574 results = CompilationResultSet()
575 timestamps = options.timestamps
576 verbose = options.verbose
577 context = None
578 for source in sources:
579 if source not in processed:
580 if context is None:
581 context = options.create_context()
582 if not timestamps or context.c_file_out_of_date(source):
583 if verbose:
584 sys.stderr.write("Compiling %s\n" % source)
586 result = run_pipeline(source, options, context=context)
587 results.add(source, result)
588 # Compiling multiple sources in one context doesn't quite
589 # work properly yet.
590 context = None
591 processed.add(source)
592 return results
594 def compile(source, options = None, full_module_name = None, **kwds):
596 compile(source [, options], [, <option> = <value>]...)
598 Compile one or more Pyrex implementation files, with optional timestamp
599 checking and recursing on dependecies. The source argument may be a string
600 or a sequence of strings If it is a string and no recursion or timestamp
601 checking is requested, a CompilationResult is returned, otherwise a
602 CompilationResultSet is returned.
604 options = CompilationOptions(defaults = options, **kwds)
605 if isinstance(source, basestring) and not options.timestamps:
606 return compile_single(source, options, full_module_name)
607 else:
608 return compile_multiple(source, options)
610 #------------------------------------------------------------------------
612 # Main command-line entry point
614 #------------------------------------------------------------------------
615 def setuptools_main():
616 return main(command_line = 1)
618 def main(command_line = 0):
619 args = sys.argv[1:]
620 any_failures = 0
621 if command_line:
622 from CmdLine import parse_command_line
623 options, sources = parse_command_line(args)
624 else:
625 options = CompilationOptions(default_options)
626 sources = args
628 if options.show_version:
629 sys.stderr.write("Cython version %s\n" % Version.version)
630 if options.working_path!="":
631 os.chdir(options.working_path)
632 try:
633 result = compile(sources, options)
634 if result.num_errors > 0:
635 any_failures = 1
636 except (EnvironmentError, PyrexError), e:
637 sys.stderr.write(str(e) + '\n')
638 any_failures = 1
639 if any_failures:
640 sys.exit(1)
644 #------------------------------------------------------------------------
646 # Set the default options depending on the platform
648 #------------------------------------------------------------------------
650 default_options = dict(
651 show_version = 0,
652 use_listing_file = 0,
653 errors_to_stderr = 1,
654 cplus = 0,
655 output_file = None,
656 annotate = None,
657 generate_pxi = 0,
658 capi_reexport_cincludes = 0,
659 working_path = "",
660 timestamps = None,
661 verbose = 0,
662 quiet = 0,
663 compiler_directives = {},
664 evaluate_tree_assertions = False,
665 emit_linenums = False,
666 relative_path_in_code_position_comments = True,
667 c_line_in_traceback = True,
668 language_level = 2,
669 gdb_debug = False,
670 compile_time_env = None,
671 common_utility_include_dir = None,