third_party/cython/src/Cython/Compiler/Parsing.py

   1 # cython: auto_cpdef=True, infer_types=True, language_level=3, py2_import=True
   2 #
   3 #   Parser
   4 #
   5
   6 # This should be done automatically
   7 import cython
   8 cython.declare(Nodes=object, ExprNodes=object, EncodedString=object,
   9                BytesLiteral=object, StringEncoding=object,
  10                FileSourceDescriptor=object, lookup_unicodechar=object,
  11                Future=object, Options=object, error=object, warning=object,
  12                Builtin=object, ModuleNode=object, Utils=object,
  13                re=object, _unicode=object, _bytes=object)
  14
  15 import re
  16 from unicodedata import lookup as lookup_unicodechar
  17
  18 from Cython.Compiler.Scanning import PyrexScanner, FileSourceDescriptor
  19 import Nodes
  20 import ExprNodes
  21 import Builtin
  22 import StringEncoding
  23 from StringEncoding import EncodedString, BytesLiteral, _unicode, _bytes
  24 from ModuleNode import ModuleNode
  25 from Errors import error, warning
  26 from Cython import Utils
  27 import Future
  28 import Options
  29
  30 class Ctx(object):
  31     #  Parsing context
  32     level = 'other'
  33     visibility = 'private'
  34     cdef_flag = 0
  35     typedef_flag = 0
  36     api = 0
  37     overridable = 0
  38     nogil = 0
  39     namespace = None
  40     templates = None
  41     allow_struct_enum_decorator = False
  42
  43     def __init__(self, **kwds):
  44         self.__dict__.update(kwds)
  45
  46     def __call__(self, **kwds):
  47         ctx = Ctx()
  48         d = ctx.__dict__
  49         d.update(self.__dict__)
  50         d.update(kwds)
  51         return ctx
  52
  53 def p_ident(s, message = "Expected an identifier"):
  54     if s.sy == 'IDENT':
  55         name = s.systring
  56         s.next()
  57         return name
  58     else:
  59         s.error(message)
  60
  61 def p_ident_list(s):
  62     names = []
  63     while s.sy == 'IDENT':
  64         names.append(s.systring)
  65         s.next()
  66         if s.sy != ',':
  67             break
  68         s.next()
  69     return names
  70
  71 #------------------------------------------
  72 #
  73 #   Expressions
  74 #
  75 #------------------------------------------
  76
  77 def p_binop_operator(s):
  78     pos = s.position()
  79     op = s.sy
  80     s.next()
  81     return op, pos
  82
  83 def p_binop_expr(s, ops, p_sub_expr):
  84     n1 = p_sub_expr(s)
  85     while s.sy in ops:
  86         op, pos = p_binop_operator(s)
  87         n2 = p_sub_expr(s)
  88         n1 = ExprNodes.binop_node(pos, op, n1, n2)
  89         if op == '/':
  90             if Future.division in s.context.future_directives:
  91                 n1.truedivision = True
  92             else:
  93                 n1.truedivision = None # unknown
  94     return n1
  95
  96 #lambdef: 'lambda' [varargslist] ':' test
  97
  98 def p_lambdef(s, allow_conditional=True):
  99     # s.sy == 'lambda'
 100     pos = s.position()
 101     s.next()
 102     if s.sy == ':':
 103         args = []
 104         star_arg = starstar_arg = None
 105     else:
 106         args, star_arg, starstar_arg = p_varargslist(
 107             s, terminator=':', annotated=False)
 108     s.expect(':')
 109     if allow_conditional:
 110         expr = p_test(s)
 111     else:
 112         expr = p_test_nocond(s)
 113     return ExprNodes.LambdaNode(
 114         pos, args = args,
 115         star_arg = star_arg, starstar_arg = starstar_arg,
 116         result_expr = expr)
 117
 118 #lambdef_nocond: 'lambda' [varargslist] ':' test_nocond
 119
 120 def p_lambdef_nocond(s):
 121     return p_lambdef(s, allow_conditional=False)
 122
 123 #test: or_test ['if' or_test 'else' test] | lambdef
 124
 125 def p_test(s):
 126     if s.sy == 'lambda':
 127         return p_lambdef(s)
 128     pos = s.position()
 129     expr = p_or_test(s)
 130     if s.sy == 'if':
 131         s.next()
 132         test = p_or_test(s)
 133         s.expect('else')
 134         other = p_test(s)
 135         return ExprNodes.CondExprNode(pos, test=test, true_val=expr, false_val=other)
 136     else:
 137         return expr
 138
 139 #test_nocond: or_test | lambdef_nocond
 140
 141 def p_test_nocond(s):
 142     if s.sy == 'lambda':
 143         return p_lambdef_nocond(s)
 144     else:
 145         return p_or_test(s)
 146
 147 #or_test: and_test ('or' and_test)*
 148
 149 def p_or_test(s):
 150     return p_rassoc_binop_expr(s, ('or',), p_and_test)
 151
 152 def p_rassoc_binop_expr(s, ops, p_subexpr):
 153     n1 = p_subexpr(s)
 154     if s.sy in ops:
 155         pos = s.position()
 156         op = s.sy
 157         s.next()
 158         n2 = p_rassoc_binop_expr(s, ops, p_subexpr)
 159         n1 = ExprNodes.binop_node(pos, op, n1, n2)
 160     return n1
 161
 162 #and_test: not_test ('and' not_test)*
 163
 164 def p_and_test(s):
 165     #return p_binop_expr(s, ('and',), p_not_test)
 166     return p_rassoc_binop_expr(s, ('and',), p_not_test)
 167
 168 #not_test: 'not' not_test | comparison
 169
 170 def p_not_test(s):
 171     if s.sy == 'not':
 172         pos = s.position()
 173         s.next()
 174         return ExprNodes.NotNode(pos, operand = p_not_test(s))
 175     else:
 176         return p_comparison(s)
 177
 178 #comparison: expr (comp_op expr)*
 179 #comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not'
 180
 181 def p_comparison(s):
 182     n1 = p_starred_expr(s)
 183     if s.sy in comparison_ops:
 184         pos = s.position()
 185         op = p_cmp_op(s)
 186         n2 = p_starred_expr(s)
 187         n1 = ExprNodes.PrimaryCmpNode(pos,
 188             operator = op, operand1 = n1, operand2 = n2)
 189         if s.sy in comparison_ops:
 190             n1.cascade = p_cascaded_cmp(s)
 191     return n1
 192
 193 def p_test_or_starred_expr(s):
 194     if s.sy == '*':
 195         return p_starred_expr(s)
 196     else:
 197         return p_test(s)
 198
 199 def p_starred_expr(s):
 200     pos = s.position()
 201     if s.sy == '*':
 202         starred = True
 203         s.next()
 204     else:
 205         starred = False
 206     expr = p_bit_expr(s)
 207     if starred:
 208         expr = ExprNodes.StarredTargetNode(pos, expr)
 209     return expr
 210
 211 def p_cascaded_cmp(s):
 212     pos = s.position()
 213     op = p_cmp_op(s)
 214     n2 = p_starred_expr(s)
 215     result = ExprNodes.CascadedCmpNode(pos,
 216         operator = op, operand2 = n2)
 217     if s.sy in comparison_ops:
 218         result.cascade = p_cascaded_cmp(s)
 219     return result
 220
 221 def p_cmp_op(s):
 222     if s.sy == 'not':
 223         s.next()
 224         s.expect('in')
 225         op = 'not_in'
 226     elif s.sy == 'is':
 227         s.next()
 228         if s.sy == 'not':
 229             s.next()
 230             op = 'is_not'
 231         else:
 232             op = 'is'
 233     else:
 234         op = s.sy
 235         s.next()
 236     if op == '<>':
 237         op = '!='
 238     return op
 239
 240 comparison_ops = cython.declare(set, set([
 241     '<', '>', '==', '>=', '<=', '<>', '!=',
 242     'in', 'is', 'not'
 243 ]))
 244
 245 #expr: xor_expr ('|' xor_expr)*
 246
 247 def p_bit_expr(s):
 248     return p_binop_expr(s, ('|',), p_xor_expr)
 249
 250 #xor_expr: and_expr ('^' and_expr)*
 251
 252 def p_xor_expr(s):
 253     return p_binop_expr(s, ('^',), p_and_expr)
 254
 255 #and_expr: shift_expr ('&' shift_expr)*
 256
 257 def p_and_expr(s):
 258     return p_binop_expr(s, ('&',), p_shift_expr)
 259
 260 #shift_expr: arith_expr (('<<'|'>>') arith_expr)*
 261
 262 def p_shift_expr(s):
 263     return p_binop_expr(s, ('<<', '>>'), p_arith_expr)
 264
 265 #arith_expr: term (('+'|'-') term)*
 266
 267 def p_arith_expr(s):
 268     return p_binop_expr(s, ('+', '-'), p_term)
 269
 270 #term: factor (('*'|'/'|'%') factor)*
 271
 272 def p_term(s):
 273     return p_binop_expr(s, ('*', '/', '%', '//'), p_factor)
 274
 275 #factor: ('+'|'-'|'~'|'&'|typecast|sizeof) factor | power
 276
 277 def p_factor(s):
 278     # little indirection for C-ification purposes
 279     return _p_factor(s)
 280
 281 def _p_factor(s):
 282     sy = s.sy
 283     if sy in ('+', '-', '~'):
 284         op = s.sy
 285         pos = s.position()
 286         s.next()
 287         return ExprNodes.unop_node(pos, op, p_factor(s))
 288     elif not s.in_python_file:
 289         if sy == '&':
 290             pos = s.position()
 291             s.next()
 292             arg = p_factor(s)
 293             return ExprNodes.AmpersandNode(pos, operand = arg)
 294         elif sy == "<":
 295             return p_typecast(s)
 296         elif sy == 'IDENT' and s.systring == "sizeof":
 297             return p_sizeof(s)
 298     return p_power(s)
 299
 300 def p_typecast(s):
 301     # s.sy == "<"
 302     pos = s.position()
 303     s.next()
 304     base_type = p_c_base_type(s)
 305     is_memslice = isinstance(base_type, Nodes.MemoryViewSliceTypeNode)
 306     is_template = isinstance(base_type, Nodes.TemplatedTypeNode)
 307     is_const = isinstance(base_type, Nodes.CConstTypeNode)
 308     if (not is_memslice and not is_template and not is_const
 309         and base_type.name is None):
 310         s.error("Unknown type")
 311     declarator = p_c_declarator(s, empty = 1)
 312     if s.sy == '?':
 313         s.next()
 314         typecheck = 1
 315     else:
 316         typecheck = 0
 317     s.expect(">")
 318     operand = p_factor(s)
 319     if is_memslice:
 320         return ExprNodes.CythonArrayNode(pos, base_type_node=base_type,
 321                                          operand=operand)
 322
 323     return ExprNodes.TypecastNode(pos,
 324         base_type = base_type,
 325         declarator = declarator,
 326         operand = operand,
 327         typecheck = typecheck)
 328
 329 def p_sizeof(s):
 330     # s.sy == ident "sizeof"
 331     pos = s.position()
 332     s.next()
 333     s.expect('(')
 334     # Here we decide if we are looking at an expression or type
 335     # If it is actually a type, but parsable as an expression,
 336     # we treat it as an expression here.
 337     if looking_at_expr(s):
 338         operand = p_test(s)
 339         node = ExprNodes.SizeofVarNode(pos, operand = operand)
 340     else:
 341         base_type = p_c_base_type(s)
 342         declarator = p_c_declarator(s, empty = 1)
 343         node = ExprNodes.SizeofTypeNode(pos,
 344             base_type = base_type, declarator = declarator)
 345     s.expect(')')
 346     return node
 347
 348 def p_yield_expression(s):
 349     # s.sy == "yield"
 350     pos = s.position()
 351     s.next()
 352     is_yield_from = False
 353     if s.sy == 'from':
 354         is_yield_from = True
 355         s.next()
 356     if s.sy != ')' and s.sy not in statement_terminators:
 357         arg = p_testlist(s)
 358     else:
 359         if is_yield_from:
 360             s.error("'yield from' requires a source argument",
 361                     pos=pos, fatal=False)
 362         arg = None
 363     if is_yield_from:
 364         return ExprNodes.YieldFromExprNode(pos, arg=arg)
 365     else:
 366         return ExprNodes.YieldExprNode(pos, arg=arg)
 367
 368 def p_yield_statement(s):
 369     # s.sy == "yield"
 370     yield_expr = p_yield_expression(s)
 371     return Nodes.ExprStatNode(yield_expr.pos, expr=yield_expr)
 372
 373 #power: atom trailer* ('**' factor)*
 374
 375 def p_power(s):
 376     if s.systring == 'new' and s.peek()[0] == 'IDENT':
 377         return p_new_expr(s)
 378     n1 = p_atom(s)
 379     while s.sy in ('(', '[', '.'):
 380         n1 = p_trailer(s, n1)
 381     if s.sy == '**':
 382         pos = s.position()
 383         s.next()
 384         n2 = p_factor(s)
 385         n1 = ExprNodes.binop_node(pos, '**', n1, n2)
 386     return n1
 387
 388 def p_new_expr(s):
 389     # s.systring == 'new'.
 390     pos = s.position()
 391     s.next()
 392     cppclass = p_c_base_type(s)
 393     return p_call(s, ExprNodes.NewExprNode(pos, cppclass = cppclass))
 394
 395 #trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME
 396
 397 def p_trailer(s, node1):
 398     pos = s.position()
 399     if s.sy == '(':
 400         return p_call(s, node1)
 401     elif s.sy == '[':
 402         return p_index(s, node1)
 403     else: # s.sy == '.'
 404         s.next()
 405         name = EncodedString( p_ident(s) )
 406         return ExprNodes.AttributeNode(pos,
 407             obj = node1, attribute = name)
 408
 409 # arglist:  argument (',' argument)* [',']
 410 # argument: [test '='] test       # Really [keyword '='] test
 411
 412 def p_call_parse_args(s, allow_genexp = True):
 413     # s.sy == '('
 414     pos = s.position()
 415     s.next()
 416     positional_args = []
 417     keyword_args = []
 418     star_arg = None
 419     starstar_arg = None
 420     while s.sy not in ('**', ')'):
 421         if s.sy == '*':
 422             if star_arg:
 423                 s.error("only one star-arg parameter allowed",
 424                         pos=s.position())
 425             s.next()
 426             star_arg = p_test(s)
 427         else:
 428             arg = p_test(s)
 429             if s.sy == '=':
 430                 s.next()
 431                 if not arg.is_name:
 432                     s.error("Expected an identifier before '='",
 433                             pos=arg.pos)
 434                 encoded_name = EncodedString(arg.name)
 435                 keyword = ExprNodes.IdentifierStringNode(
 436                     arg.pos, value=encoded_name)
 437                 arg = p_test(s)
 438                 keyword_args.append((keyword, arg))
 439             else:
 440                 if keyword_args:
 441                     s.error("Non-keyword arg following keyword arg",
 442                             pos=arg.pos)
 443                 if star_arg:
 444                     s.error("Non-keyword arg following star-arg",
 445                             pos=arg.pos)
 446                 positional_args.append(arg)
 447         if s.sy != ',':
 448             break
 449         s.next()
 450
 451     if s.sy == 'for':
 452         if len(positional_args) == 1 and not star_arg:
 453             positional_args = [ p_genexp(s, positional_args[0]) ]
 454     elif s.sy == '**':
 455         s.next()
 456         starstar_arg = p_test(s)
 457         if s.sy == ',':
 458             s.next()
 459     s.expect(')')
 460     return positional_args, keyword_args, star_arg, starstar_arg
 461
 462 def p_call_build_packed_args(pos, positional_args, keyword_args,
 463                              star_arg, starstar_arg):
 464     arg_tuple = None
 465     keyword_dict = None
 466     if positional_args or not star_arg:
 467         arg_tuple = ExprNodes.TupleNode(pos,
 468             args = positional_args)
 469     if star_arg:
 470         star_arg_tuple = ExprNodes.AsTupleNode(pos, arg = star_arg)
 471         if arg_tuple:
 472             arg_tuple = ExprNodes.binop_node(pos,
 473                 operator = '+', operand1 = arg_tuple,
 474                 operand2 = star_arg_tuple)
 475         else:
 476             arg_tuple = star_arg_tuple
 477     if keyword_args or starstar_arg:
 478         keyword_args = [ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
 479                           for key, value in keyword_args]
 480         if starstar_arg:
 481             keyword_dict = ExprNodes.KeywordArgsNode(
 482                 pos,
 483                 starstar_arg = starstar_arg,
 484                 keyword_args = keyword_args)
 485         else:
 486             keyword_dict = ExprNodes.DictNode(
 487                 pos, key_value_pairs = keyword_args)
 488     return arg_tuple, keyword_dict
 489
 490 def p_call(s, function):
 491     # s.sy == '('
 492     pos = s.position()
 493
 494     positional_args, keyword_args, star_arg, starstar_arg = \
 495                      p_call_parse_args(s)
 496
 497     if not (keyword_args or star_arg or starstar_arg):
 498         return ExprNodes.SimpleCallNode(pos,
 499             function = function,
 500             args = positional_args)
 501     else:
 502         arg_tuple, keyword_dict = p_call_build_packed_args(
 503             pos, positional_args, keyword_args, star_arg, starstar_arg)
 504         return ExprNodes.GeneralCallNode(pos,
 505             function = function,
 506             positional_args = arg_tuple,
 507             keyword_args = keyword_dict)
 508
 509 #lambdef: 'lambda' [varargslist] ':' test
 510
 511 #subscriptlist: subscript (',' subscript)* [',']
 512
 513 def p_index(s, base):
 514     # s.sy == '['
 515     pos = s.position()
 516     s.next()
 517     subscripts, is_single_value = p_subscript_list(s)
 518     if is_single_value and len(subscripts[0]) == 2:
 519         start, stop = subscripts[0]
 520         result = ExprNodes.SliceIndexNode(pos,
 521             base = base, start = start, stop = stop)
 522     else:
 523         indexes = make_slice_nodes(pos, subscripts)
 524         if is_single_value:
 525             index = indexes[0]
 526         else:
 527             index = ExprNodes.TupleNode(pos, args = indexes)
 528         result = ExprNodes.IndexNode(pos,
 529             base = base, index = index)
 530     s.expect(']')
 531     return result
 532
 533 def p_subscript_list(s):
 534     is_single_value = True
 535     items = [p_subscript(s)]
 536     while s.sy == ',':
 537         is_single_value = False
 538         s.next()
 539         if s.sy == ']':
 540             break
 541         items.append(p_subscript(s))
 542     return items, is_single_value
 543
 544 #subscript: '.' '.' '.' | test | [test] ':' [test] [':' [test]]
 545
 546 def p_subscript(s):
 547     # Parse a subscript and return a list of
 548     # 1, 2 or 3 ExprNodes, depending on how
 549     # many slice elements were encountered.
 550     pos = s.position()
 551     start = p_slice_element(s, (':',))
 552     if s.sy != ':':
 553         return [start]
 554     s.next()
 555     stop = p_slice_element(s, (':', ',', ']'))
 556     if s.sy != ':':
 557         return [start, stop]
 558     s.next()
 559     step = p_slice_element(s, (':', ',', ']'))
 560     return [start, stop, step]
 561
 562 def p_slice_element(s, follow_set):
 563     # Simple expression which may be missing iff
 564     # it is followed by something in follow_set.
 565     if s.sy not in follow_set:
 566         return p_test(s)
 567     else:
 568         return None
 569
 570 def expect_ellipsis(s):
 571     s.expect('.')
 572     s.expect('.')
 573     s.expect('.')
 574
 575 def make_slice_nodes(pos, subscripts):
 576     # Convert a list of subscripts as returned
 577     # by p_subscript_list into a list of ExprNodes,
 578     # creating SliceNodes for elements with 2 or
 579     # more components.
 580     result = []
 581     for subscript in subscripts:
 582         if len(subscript) == 1:
 583             result.append(subscript[0])
 584         else:
 585             result.append(make_slice_node(pos, *subscript))
 586     return result
 587
 588 def make_slice_node(pos, start, stop = None, step = None):
 589     if not start:
 590         start = ExprNodes.NoneNode(pos)
 591     if not stop:
 592         stop = ExprNodes.NoneNode(pos)
 593     if not step:
 594         step = ExprNodes.NoneNode(pos)
 595     return ExprNodes.SliceNode(pos,
 596         start = start, stop = stop, step = step)
 597
 598 #atom: '(' [yield_expr|testlist_comp] ')' | '[' [listmaker] ']' | '{' [dict_or_set_maker] '}' | '`' testlist '`' | NAME | NUMBER | STRING+
 599
 600 def p_atom(s):
 601     pos = s.position()
 602     sy = s.sy
 603     if sy == '(':
 604         s.next()
 605         if s.sy == ')':
 606             result = ExprNodes.TupleNode(pos, args = [])
 607         elif s.sy == 'yield':
 608             result = p_yield_expression(s)
 609         else:
 610             result = p_testlist_comp(s)
 611         s.expect(')')
 612         return result
 613     elif sy == '[':
 614         return p_list_maker(s)
 615     elif sy == '{':
 616         return p_dict_or_set_maker(s)
 617     elif sy == '`':
 618         return p_backquote_expr(s)
 619     elif sy == '.':
 620         expect_ellipsis(s)
 621         return ExprNodes.EllipsisNode(pos)
 622     elif sy == 'INT':
 623         return p_int_literal(s)
 624     elif sy == 'FLOAT':
 625         value = s.systring
 626         s.next()
 627         return ExprNodes.FloatNode(pos, value = value)
 628     elif sy == 'IMAG':
 629         value = s.systring[:-1]
 630         s.next()
 631         return ExprNodes.ImagNode(pos, value = value)
 632     elif sy == 'BEGIN_STRING':
 633         kind, bytes_value, unicode_value = p_cat_string_literal(s)
 634         if kind == 'c':
 635             return ExprNodes.CharNode(pos, value = bytes_value)
 636         elif kind == 'u':
 637             return ExprNodes.UnicodeNode(pos, value = unicode_value, bytes_value = bytes_value)
 638         elif kind == 'b':
 639             return ExprNodes.BytesNode(pos, value = bytes_value)
 640         else:
 641             return ExprNodes.StringNode(pos, value = bytes_value, unicode_value = unicode_value)
 642     elif sy == 'IDENT':
 643         name = EncodedString( s.systring )
 644         s.next()
 645         if name == "None":
 646             return ExprNodes.NoneNode(pos)
 647         elif name == "True":
 648             return ExprNodes.BoolNode(pos, value=True)
 649         elif name == "False":
 650             return ExprNodes.BoolNode(pos, value=False)
 651         elif name == "NULL" and not s.in_python_file:
 652             return ExprNodes.NullNode(pos)
 653         else:
 654             return p_name(s, name)
 655     else:
 656         s.error("Expected an identifier or literal")
 657
 658 def p_int_literal(s):
 659     pos = s.position()
 660     value = s.systring
 661     s.next()
 662     unsigned = ""
 663     longness = ""
 664     while value[-1] in u"UuLl":
 665         if value[-1] in u"Ll":
 666             longness += "L"
 667         else:
 668             unsigned += "U"
 669         value = value[:-1]
 670     # '3L' is ambiguous in Py2 but not in Py3.  '3U' and '3LL' are
 671     # illegal in Py2 Python files.  All suffixes are illegal in Py3
 672     # Python files.
 673     is_c_literal = None
 674     if unsigned:
 675         is_c_literal = True
 676     elif longness:
 677         if longness == 'LL' or s.context.language_level >= 3:
 678             is_c_literal = True
 679     if s.in_python_file:
 680         if is_c_literal:
 681             error(pos, "illegal integer literal syntax in Python source file")
 682         is_c_literal = False
 683     return ExprNodes.IntNode(pos,
 684                              is_c_literal = is_c_literal,
 685                              value = value,
 686                              unsigned = unsigned,
 687                              longness = longness)
 688
 689
 690 def p_name(s, name):
 691     pos = s.position()
 692     if not s.compile_time_expr and name in s.compile_time_env:
 693         value = s.compile_time_env.lookup_here(name)
 694         node = wrap_compile_time_constant(pos, value)
 695         if node is not None:
 696             return node
 697     return ExprNodes.NameNode(pos, name=name)
 698
 699
 700 def wrap_compile_time_constant(pos, value):
 701     rep = repr(value)
 702     if value is None:
 703         return ExprNodes.NoneNode(pos)
 704     elif value is Ellipsis:
 705         return ExprNodes.EllipsisNode(pos)
 706     elif isinstance(value, bool):
 707         return ExprNodes.BoolNode(pos, value=value)
 708     elif isinstance(value, int):
 709         return ExprNodes.IntNode(pos, value=rep)
 710     elif isinstance(value, long):
 711         return ExprNodes.IntNode(pos, value=rep, longness="L")
 712     elif isinstance(value, float):
 713         return ExprNodes.FloatNode(pos, value=rep)
 714     elif isinstance(value, _unicode):
 715         return ExprNodes.UnicodeNode(pos, value=EncodedString(value))
 716     elif isinstance(value, _bytes):
 717         return ExprNodes.BytesNode(pos, value=BytesLiteral(value))
 718     elif isinstance(value, tuple):
 719         args = [wrap_compile_time_constant(pos, arg)
 720                 for arg in value]
 721         if None not in args:
 722             return ExprNodes.TupleNode(pos, args=args)
 723         else:
 724             # error already reported
 725             return None
 726     error(pos, "Invalid type for compile-time constant: %r (type %s)"
 727                % (value, value.__class__.__name__))
 728     return None
 729
 730
 731 def p_cat_string_literal(s):
 732     # A sequence of one or more adjacent string literals.
 733     # Returns (kind, bytes_value, unicode_value)
 734     # where kind in ('b', 'c', 'u', '')
 735     kind, bytes_value, unicode_value = p_string_literal(s)
 736     if kind == 'c' or s.sy != 'BEGIN_STRING':
 737         return kind, bytes_value, unicode_value
 738     bstrings, ustrings = [bytes_value], [unicode_value]
 739     bytes_value = unicode_value = None
 740     while s.sy == 'BEGIN_STRING':
 741         pos = s.position()
 742         next_kind, next_bytes_value, next_unicode_value = p_string_literal(s)
 743         if next_kind == 'c':
 744             error(pos, "Cannot concatenate char literal with another string or char literal")
 745         elif next_kind != kind:
 746             error(pos, "Cannot mix string literals of different types, expected %s'', got %s''" %
 747                   (kind, next_kind))
 748         else:
 749             bstrings.append(next_bytes_value)
 750             ustrings.append(next_unicode_value)
 751     # join and rewrap the partial literals
 752     if kind in ('b', 'c', '') or kind == 'u' and None not in bstrings:
 753         # Py3 enforced unicode literals are parsed as bytes/unicode combination
 754         bytes_value = BytesLiteral( StringEncoding.join_bytes(bstrings) )
 755         bytes_value.encoding = s.source_encoding
 756     if kind in ('u', ''):
 757         unicode_value = EncodedString( u''.join([ u for u in ustrings if u is not None ]) )
 758     return kind, bytes_value, unicode_value
 759
 760 def p_opt_string_literal(s, required_type='u'):
 761     if s.sy == 'BEGIN_STRING':
 762         kind, bytes_value, unicode_value = p_string_literal(s, required_type)
 763         if required_type == 'u':
 764             return unicode_value
 765         elif required_type == 'b':
 766             return bytes_value
 767         else:
 768             s.error("internal parser configuration error")
 769     else:
 770         return None
 771
 772 def check_for_non_ascii_characters(string):
 773     for c in string:
 774         if c >= u'\x80':
 775             return True
 776     return False
 777
 778 def p_string_literal(s, kind_override=None):
 779     # A single string or char literal.  Returns (kind, bvalue, uvalue)
 780     # where kind in ('b', 'c', 'u', '').  The 'bvalue' is the source
 781     # code byte sequence of the string literal, 'uvalue' is the
 782     # decoded Unicode string.  Either of the two may be None depending
 783     # on the 'kind' of string, only unprefixed strings have both
 784     # representations.
 785
 786     # s.sy == 'BEGIN_STRING'
 787     pos = s.position()
 788     is_raw = False
 789     is_python3_source = s.context.language_level >= 3
 790     has_non_ASCII_literal_characters = False
 791     kind = s.systring[:1].lower()
 792     if kind == 'r':
 793         # Py3 allows both 'br' and 'rb' as prefix
 794         if s.systring[1:2].lower() == 'b':
 795             kind = 'b'
 796         else:
 797             kind = ''
 798         is_raw = True
 799     elif kind in 'ub':
 800         is_raw = s.systring[1:2].lower() == 'r'
 801     elif kind != 'c':
 802         kind = ''
 803     if kind == '' and kind_override is None and Future.unicode_literals in s.context.future_directives:
 804         chars = StringEncoding.StrLiteralBuilder(s.source_encoding)
 805         kind = 'u'
 806     else:
 807         if kind_override is not None and kind_override in 'ub':
 808             kind = kind_override
 809         if kind == 'u':
 810             chars = StringEncoding.UnicodeLiteralBuilder()
 811         elif kind == '':
 812             chars = StringEncoding.StrLiteralBuilder(s.source_encoding)
 813         else:
 814             chars = StringEncoding.BytesLiteralBuilder(s.source_encoding)
 815
 816     while 1:
 817         s.next()
 818         sy = s.sy
 819         systr = s.systring
 820         #print "p_string_literal: sy =", sy, repr(s.systring) ###
 821         if sy == 'CHARS':
 822             chars.append(systr)
 823             if is_python3_source and not has_non_ASCII_literal_characters and check_for_non_ascii_characters(systr):
 824                 has_non_ASCII_literal_characters = True
 825         elif sy == 'ESCAPE':
 826             if is_raw:
 827                 chars.append(systr)
 828                 if is_python3_source and not has_non_ASCII_literal_characters \
 829                        and check_for_non_ascii_characters(systr):
 830                     has_non_ASCII_literal_characters = True
 831             else:
 832                 c = systr[1]
 833                 if c in u"01234567":
 834                     chars.append_charval( int(systr[1:], 8) )
 835                 elif c in u"'\"\\":
 836                     chars.append(c)
 837                 elif c in u"abfnrtv":
 838                     chars.append(
 839                         StringEncoding.char_from_escape_sequence(systr))
 840                 elif c == u'\n':
 841                     pass
 842                 elif c == u'x':   # \xXX
 843                     if len(systr) == 4:
 844                         chars.append_charval( int(systr[2:], 16) )
 845                     else:
 846                         s.error("Invalid hex escape '%s'" % systr,
 847                                 fatal=False)
 848                 elif c in u'NUu' and kind in ('u', ''):   # \uxxxx, \Uxxxxxxxx, \N{...}
 849                     chrval = -1
 850                     if c == u'N':
 851                         try:
 852                             chrval = ord(lookup_unicodechar(systr[3:-1]))
 853                         except KeyError:
 854                             s.error("Unknown Unicode character name %s" %
 855                                     repr(systr[3:-1]).lstrip('u'))
 856                     elif len(systr) in (6,10):
 857                         chrval = int(systr[2:], 16)
 858                         if chrval > 1114111: # sys.maxunicode:
 859                             s.error("Invalid unicode escape '%s'" % systr)
 860                             chrval = -1
 861                     else:
 862                         s.error("Invalid unicode escape '%s'" % systr,
 863                                 fatal=False)
 864                     if chrval >= 0:
 865                         chars.append_uescape(chrval, systr)
 866                 else:
 867                     chars.append(u'\\' + systr[1:])
 868                     if is_python3_source and not has_non_ASCII_literal_characters \
 869                            and check_for_non_ascii_characters(systr):
 870                         has_non_ASCII_literal_characters = True
 871         elif sy == 'NEWLINE':
 872             chars.append(u'\n')
 873         elif sy == 'END_STRING':
 874             break
 875         elif sy == 'EOF':
 876             s.error("Unclosed string literal", pos=pos)
 877         else:
 878             s.error("Unexpected token %r:%r in string literal" %
 879                     (sy, s.systring))
 880
 881     if kind == 'c':
 882         unicode_value = None
 883         bytes_value = chars.getchar()
 884         if len(bytes_value) != 1:
 885             error(pos, u"invalid character literal: %r" % bytes_value)
 886     else:
 887         bytes_value, unicode_value = chars.getstrings()
 888         if is_python3_source and has_non_ASCII_literal_characters:
 889             # Python 3 forbids literal non-ASCII characters in byte strings
 890             if kind != 'u':
 891                 s.error("bytes can only contain ASCII literal characters.",
 892                         pos=pos, fatal=False)
 893             bytes_value = None
 894     s.next()
 895     return (kind, bytes_value, unicode_value)
 896
 897 # list_display      ::=      "[" [listmaker] "]"
 898 # listmaker     ::=     expression ( comp_for | ( "," expression )* [","] )
 899 # comp_iter     ::=     comp_for | comp_if
 900 # comp_for     ::=     "for" expression_list "in" testlist [comp_iter]
 901 # comp_if     ::=     "if" test [comp_iter]
 902
 903 def p_list_maker(s):
 904     # s.sy == '['
 905     pos = s.position()
 906     s.next()
 907     if s.sy == ']':
 908         s.expect(']')
 909         return ExprNodes.ListNode(pos, args = [])
 910     expr = p_test(s)
 911     if s.sy == 'for':
 912         append = ExprNodes.ComprehensionAppendNode(pos, expr=expr)
 913         loop = p_comp_for(s, append)
 914         s.expect(']')
 915         return ExprNodes.ComprehensionNode(
 916             pos, loop=loop, append=append, type = Builtin.list_type,
 917             # list comprehensions leak their loop variable in Py2
 918             has_local_scope = s.context.language_level >= 3)
 919     else:
 920         if s.sy == ',':
 921             s.next()
 922             exprs = p_simple_expr_list(s, expr)
 923         else:
 924             exprs = [expr]
 925         s.expect(']')
 926         return ExprNodes.ListNode(pos, args = exprs)
 927
 928 def p_comp_iter(s, body):
 929     if s.sy == 'for':
 930         return p_comp_for(s, body)
 931     elif s.sy == 'if':
 932         return p_comp_if(s, body)
 933     else:
 934         # insert the 'append' operation into the loop
 935         return body
 936
 937 def p_comp_for(s, body):
 938     # s.sy == 'for'
 939     pos = s.position()
 940     s.next()
 941     kw = p_for_bounds(s, allow_testlist=False)
 942     kw.update(else_clause = None, body = p_comp_iter(s, body))
 943     return Nodes.ForStatNode(pos, **kw)
 944
 945 def p_comp_if(s, body):
 946     # s.sy == 'if'
 947     pos = s.position()
 948     s.next()
 949     test = p_test_nocond(s)
 950     return Nodes.IfStatNode(pos,
 951         if_clauses = [Nodes.IfClauseNode(pos, condition = test,
 952                                          body = p_comp_iter(s, body))],
 953         else_clause = None )
 954
 955 #dictmaker: test ':' test (',' test ':' test)* [',']
 956
 957 def p_dict_or_set_maker(s):
 958     # s.sy == '{'
 959     pos = s.position()
 960     s.next()
 961     if s.sy == '}':
 962         s.next()
 963         return ExprNodes.DictNode(pos, key_value_pairs = [])
 964     item = p_test(s)
 965     if s.sy == ',' or s.sy == '}':
 966         # set literal
 967         values = [item]
 968         while s.sy == ',':
 969             s.next()
 970             if s.sy == '}':
 971                 break
 972             values.append( p_test(s) )
 973         s.expect('}')
 974         return ExprNodes.SetNode(pos, args=values)
 975     elif s.sy == 'for':
 976         # set comprehension
 977         append = ExprNodes.ComprehensionAppendNode(
 978             item.pos, expr=item)
 979         loop = p_comp_for(s, append)
 980         s.expect('}')
 981         return ExprNodes.ComprehensionNode(
 982             pos, loop=loop, append=append, type=Builtin.set_type)
 983     elif s.sy == ':':
 984         # dict literal or comprehension
 985         key = item
 986         s.next()
 987         value = p_test(s)
 988         if s.sy == 'for':
 989             # dict comprehension
 990             append = ExprNodes.DictComprehensionAppendNode(
 991                 item.pos, key_expr=key, value_expr=value)
 992             loop = p_comp_for(s, append)
 993             s.expect('}')
 994             return ExprNodes.ComprehensionNode(
 995                 pos, loop=loop, append=append, type=Builtin.dict_type)
 996         else:
 997             # dict literal
 998             items = [ExprNodes.DictItemNode(key.pos, key=key, value=value)]
 999             while s.sy == ',':
1000                 s.next()
1001                 if s.sy == '}':
1002                     break
1003                 key = p_test(s)
1004                 s.expect(':')
1005                 value = p_test(s)
1006                 items.append(
1007                     ExprNodes.DictItemNode(key.pos, key=key, value=value))
1008             s.expect('}')
1009             return ExprNodes.DictNode(pos, key_value_pairs=items)
1010     else:
1011         # raise an error
1012         s.expect('}')
1013     return ExprNodes.DictNode(pos, key_value_pairs = [])
1014
1015 # NOTE: no longer in Py3 :)
1016 def p_backquote_expr(s):
1017     # s.sy == '`'
1018     pos = s.position()
1019     s.next()
1020     args = [p_test(s)]
1021     while s.sy == ',':
1022         s.next()
1023         args.append(p_test(s))
1024     s.expect('`')
1025     if len(args) == 1:
1026         arg = args[0]
1027     else:
1028         arg = ExprNodes.TupleNode(pos, args = args)
1029     return ExprNodes.BackquoteNode(pos, arg = arg)
1030
1031 def p_simple_expr_list(s, expr=None):
1032     exprs = expr is not None and [expr] or []
1033     while s.sy not in expr_terminators:
1034         exprs.append( p_test(s) )
1035         if s.sy != ',':
1036             break
1037         s.next()
1038     return exprs
1039
1040 def p_test_or_starred_expr_list(s, expr=None):
1041     exprs = expr is not None and [expr] or []
1042     while s.sy not in expr_terminators:
1043         exprs.append( p_test_or_starred_expr(s) )
1044         if s.sy != ',':
1045             break
1046         s.next()
1047     return exprs
1048
1049
1050 #testlist: test (',' test)* [',']
1051
1052 def p_testlist(s):
1053     pos = s.position()
1054     expr = p_test(s)
1055     if s.sy == ',':
1056         s.next()
1057         exprs = p_simple_expr_list(s, expr)
1058         return ExprNodes.TupleNode(pos, args = exprs)
1059     else:
1060         return expr
1061
1062 # testlist_star_expr: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
1063
1064 def p_testlist_star_expr(s):
1065     pos = s.position()
1066     expr = p_test_or_starred_expr(s)
1067     if s.sy == ',':
1068         s.next()
1069         exprs = p_test_or_starred_expr_list(s, expr)
1070         return ExprNodes.TupleNode(pos, args = exprs)
1071     else:
1072         return expr
1073
1074 # testlist_comp: (test|star_expr) ( comp_for | (',' (test|star_expr))* [','] )
1075
1076 def p_testlist_comp(s):
1077     pos = s.position()
1078     expr = p_test_or_starred_expr(s)
1079     if s.sy == ',':
1080         s.next()
1081         exprs = p_test_or_starred_expr_list(s, expr)
1082         return ExprNodes.TupleNode(pos, args = exprs)
1083     elif s.sy == 'for':
1084         return p_genexp(s, expr)
1085     else:
1086         return expr
1087
1088 def p_genexp(s, expr):
1089     # s.sy == 'for'
1090     loop = p_comp_for(s, Nodes.ExprStatNode(
1091         expr.pos, expr = ExprNodes.YieldExprNode(expr.pos, arg=expr)))
1092     return ExprNodes.GeneratorExpressionNode(expr.pos, loop=loop)
1093
1094 expr_terminators = cython.declare(set, set([
1095     ')', ']', '}', ':', '=', 'NEWLINE']))
1096
1097 #-------------------------------------------------------
1098 #
1099 #   Statements
1100 #
1101 #-------------------------------------------------------
1102
1103 def p_global_statement(s):
1104     # assume s.sy == 'global'
1105     pos = s.position()
1106     s.next()
1107     names = p_ident_list(s)
1108     return Nodes.GlobalNode(pos, names = names)
1109
1110 def p_nonlocal_statement(s):
1111     pos = s.position()
1112     s.next()
1113     names = p_ident_list(s)
1114     return Nodes.NonlocalNode(pos, names = names)
1115
1116 def p_expression_or_assignment(s):
1117     expr_list = [p_testlist_star_expr(s)]
1118     if s.sy == '=' and expr_list[0].is_starred:
1119         # This is a common enough error to make when learning Cython to let
1120         # it fail as early as possible and give a very clear error message.
1121         s.error("a starred assignment target must be in a list or tuple"
1122                 " - maybe you meant to use an index assignment: var[0] = ...",
1123                 pos=expr_list[0].pos)
1124     while s.sy == '=':
1125         s.next()
1126         if s.sy == 'yield':
1127             expr = p_yield_expression(s)
1128         else:
1129             expr = p_testlist_star_expr(s)
1130         expr_list.append(expr)
1131     if len(expr_list) == 1:
1132         if re.match(r"([+*/\%^\&|-]|<<|>>|\*\*|//)=", s.sy):
1133             lhs = expr_list[0]
1134             if isinstance(lhs, ExprNodes.SliceIndexNode):
1135                 # implementation requires IndexNode
1136                 lhs = ExprNodes.IndexNode(
1137                     lhs.pos,
1138                     base=lhs.base,
1139                     index=make_slice_node(lhs.pos, lhs.start, lhs.stop))
1140             elif not isinstance(lhs, (ExprNodes.AttributeNode, ExprNodes.IndexNode, ExprNodes.NameNode) ):
1141                 error(lhs.pos, "Illegal operand for inplace operation.")
1142             operator = s.sy[:-1]
1143             s.next()
1144             if s.sy == 'yield':
1145                 rhs = p_yield_expression(s)
1146             else:
1147                 rhs = p_testlist(s)
1148             return Nodes.InPlaceAssignmentNode(lhs.pos, operator = operator, lhs = lhs, rhs = rhs)
1149         expr = expr_list[0]
1150         return Nodes.ExprStatNode(expr.pos, expr=expr)
1151
1152     rhs = expr_list[-1]
1153     if len(expr_list) == 2:
1154         return Nodes.SingleAssignmentNode(rhs.pos,
1155             lhs = expr_list[0], rhs = rhs)
1156     else:
1157         return Nodes.CascadedAssignmentNode(rhs.pos,
1158             lhs_list = expr_list[:-1], rhs = rhs)
1159
1160 def p_print_statement(s):
1161     # s.sy == 'print'
1162     pos = s.position()
1163     ends_with_comma = 0
1164     s.next()
1165     if s.sy == '>>':
1166         s.next()
1167         stream = p_test(s)
1168         if s.sy == ',':
1169             s.next()
1170             ends_with_comma = s.sy in ('NEWLINE', 'EOF')
1171     else:
1172         stream = None
1173     args = []
1174     if s.sy not in ('NEWLINE', 'EOF'):
1175         args.append(p_test(s))
1176         while s.sy == ',':
1177             s.next()
1178             if s.sy in ('NEWLINE', 'EOF'):
1179                 ends_with_comma = 1
1180                 break
1181             args.append(p_test(s))
1182     arg_tuple = ExprNodes.TupleNode(pos, args = args)
1183     return Nodes.PrintStatNode(pos,
1184         arg_tuple = arg_tuple, stream = stream,
1185         append_newline = not ends_with_comma)
1186
1187 def p_exec_statement(s):
1188     # s.sy == 'exec'
1189     pos = s.position()
1190     s.next()
1191     code = p_bit_expr(s)
1192     if isinstance(code, ExprNodes.TupleNode):
1193         # Py3 compatibility syntax
1194         tuple_variant = True
1195         args = code.args
1196         if len(args) not in (2, 3):
1197             s.error("expected tuple of length 2 or 3, got length %d" % len(args),
1198                     pos=pos, fatal=False)
1199             args = [code]
1200     else:
1201         tuple_variant = False
1202         args = [code]
1203     if s.sy == 'in':
1204         if tuple_variant:
1205             s.error("tuple variant of exec does not support additional 'in' arguments",
1206                     fatal=False)
1207         s.next()
1208         args.append(p_test(s))
1209         if s.sy == ',':
1210             s.next()
1211             args.append(p_test(s))
1212     return Nodes.ExecStatNode(pos, args=args)
1213
1214 def p_del_statement(s):
1215     # s.sy == 'del'
1216     pos = s.position()
1217     s.next()
1218     # FIXME: 'exprlist' in Python
1219     args = p_simple_expr_list(s)
1220     return Nodes.DelStatNode(pos, args = args)
1221
1222 def p_pass_statement(s, with_newline = 0):
1223     pos = s.position()
1224     s.expect('pass')
1225     if with_newline:
1226         s.expect_newline("Expected a newline")
1227     return Nodes.PassStatNode(pos)
1228
1229 def p_break_statement(s):
1230     # s.sy == 'break'
1231     pos = s.position()
1232     s.next()
1233     return Nodes.BreakStatNode(pos)
1234
1235 def p_continue_statement(s):
1236     # s.sy == 'continue'
1237     pos = s.position()
1238     s.next()
1239     return Nodes.ContinueStatNode(pos)
1240
1241 def p_return_statement(s):
1242     # s.sy == 'return'
1243     pos = s.position()
1244     s.next()
1245     if s.sy not in statement_terminators:
1246         value = p_testlist(s)
1247     else:
1248         value = None
1249     return Nodes.ReturnStatNode(pos, value = value)
1250
1251 def p_raise_statement(s):
1252     # s.sy == 'raise'
1253     pos = s.position()
1254     s.next()
1255     exc_type = None
1256     exc_value = None
1257     exc_tb = None
1258     cause = None
1259     if s.sy not in statement_terminators:
1260         exc_type = p_test(s)
1261         if s.sy == ',':
1262             s.next()
1263             exc_value = p_test(s)
1264             if s.sy == ',':
1265                 s.next()
1266                 exc_tb = p_test(s)
1267         elif s.sy == 'from':
1268             s.next()
1269             cause = p_test(s)
1270     if exc_type or exc_value or exc_tb:
1271         return Nodes.RaiseStatNode(pos,
1272             exc_type = exc_type,
1273             exc_value = exc_value,
1274             exc_tb = exc_tb,
1275             cause = cause)
1276     else:
1277         return Nodes.ReraiseStatNode(pos)
1278
1279 def p_import_statement(s):
1280     # s.sy in ('import', 'cimport')
1281     pos = s.position()
1282     kind = s.sy
1283     s.next()
1284     items = [p_dotted_name(s, as_allowed = 1)]
1285     while s.sy == ',':
1286         s.next()
1287         items.append(p_dotted_name(s, as_allowed = 1))
1288     stats = []
1289     for pos, target_name, dotted_name, as_name in items:
1290         dotted_name = EncodedString(dotted_name)
1291         if kind == 'cimport':
1292             stat = Nodes.CImportStatNode(pos,
1293                 module_name = dotted_name,
1294                 as_name = as_name)
1295         else:
1296             if as_name and "." in dotted_name:
1297                 name_list = ExprNodes.ListNode(pos, args = [
1298                         ExprNodes.IdentifierStringNode(pos, value = EncodedString("*"))])
1299             else:
1300                 name_list = None
1301             stat = Nodes.SingleAssignmentNode(pos,
1302                 lhs = ExprNodes.NameNode(pos,
1303                     name = as_name or target_name),
1304                 rhs = ExprNodes.ImportNode(pos,
1305                     module_name = ExprNodes.IdentifierStringNode(
1306                         pos, value = dotted_name),
1307                     level = None,
1308                     name_list = name_list))
1309         stats.append(stat)
1310     return Nodes.StatListNode(pos, stats = stats)
1311
1312 def p_from_import_statement(s, first_statement = 0):
1313     # s.sy == 'from'
1314     pos = s.position()
1315     s.next()
1316     if s.sy == '.':
1317         # count relative import level
1318         level = 0
1319         while s.sy == '.':
1320             level += 1
1321             s.next()
1322         if s.sy == 'cimport':
1323             s.error("Relative cimport is not supported yet")
1324     else:
1325         level = None
1326     if level is not None and s.sy == 'import':
1327         # we are dealing with "from .. import foo, bar"
1328         dotted_name_pos, dotted_name = s.position(), ''
1329     elif level is not None and s.sy == 'cimport':
1330         # "from .. cimport"
1331         s.error("Relative cimport is not supported yet")
1332     else:
1333         (dotted_name_pos, _, dotted_name, _) = \
1334             p_dotted_name(s, as_allowed = 0)
1335     if s.sy in ('import', 'cimport'):
1336         kind = s.sy
1337         s.next()
1338     else:
1339         s.error("Expected 'import' or 'cimport'")
1340
1341     is_cimport = kind == 'cimport'
1342     is_parenthesized = False
1343     if s.sy == '*':
1344         imported_names = [(s.position(), "*", None, None)]
1345         s.next()
1346     else:
1347         if s.sy == '(':
1348             is_parenthesized = True
1349             s.next()
1350         imported_names = [p_imported_name(s, is_cimport)]
1351     while s.sy == ',':
1352         s.next()
1353         if is_parenthesized and s.sy == ')':
1354             break
1355         imported_names.append(p_imported_name(s, is_cimport))
1356     if is_parenthesized:
1357         s.expect(')')
1358     dotted_name = EncodedString(dotted_name)
1359     if dotted_name == '__future__':
1360         if not first_statement:
1361             s.error("from __future__ imports must occur at the beginning of the file")
1362         elif level is not None:
1363             s.error("invalid syntax")
1364         else:
1365             for (name_pos, name, as_name, kind) in imported_names:
1366                 if name == "braces":
1367                     s.error("not a chance", name_pos)
1368                     break
1369                 try:
1370                     directive = getattr(Future, name)
1371                 except AttributeError:
1372                     s.error("future feature %s is not defined" % name, name_pos)
1373                     break
1374                 s.context.future_directives.add(directive)
1375         return Nodes.PassStatNode(pos)
1376     elif kind == 'cimport':
1377         return Nodes.FromCImportStatNode(pos,
1378             module_name = dotted_name,
1379             imported_names = imported_names)
1380     else:
1381         imported_name_strings = []
1382         items = []
1383         for (name_pos, name, as_name, kind) in imported_names:
1384             encoded_name = EncodedString(name)
1385             imported_name_strings.append(
1386                 ExprNodes.IdentifierStringNode(name_pos, value = encoded_name))
1387             items.append(
1388                 (name,
1389                  ExprNodes.NameNode(name_pos,
1390                                     name = as_name or name)))
1391         import_list = ExprNodes.ListNode(
1392             imported_names[0][0], args = imported_name_strings)
1393         dotted_name = EncodedString(dotted_name)
1394         return Nodes.FromImportStatNode(pos,
1395             module = ExprNodes.ImportNode(dotted_name_pos,
1396                 module_name = ExprNodes.IdentifierStringNode(pos, value = dotted_name),
1397                 level = level,
1398                 name_list = import_list),
1399             items = items)
1400
1401 imported_name_kinds = cython.declare(
1402     set, set(['class', 'struct', 'union']))
1403
1404 def p_imported_name(s, is_cimport):
1405     pos = s.position()
1406     kind = None
1407     if is_cimport and s.systring in imported_name_kinds:
1408         kind = s.systring
1409         s.next()
1410     name = p_ident(s)
1411     as_name = p_as_name(s)
1412     return (pos, name, as_name, kind)
1413
1414 def p_dotted_name(s, as_allowed):
1415     pos = s.position()
1416     target_name = p_ident(s)
1417     as_name = None
1418     names = [target_name]
1419     while s.sy == '.':
1420         s.next()
1421         names.append(p_ident(s))
1422     if as_allowed:
1423         as_name = p_as_name(s)
1424     return (pos, target_name, u'.'.join(names), as_name)
1425
1426 def p_as_name(s):
1427     if s.sy == 'IDENT' and s.systring == 'as':
1428         s.next()
1429         return p_ident(s)
1430     else:
1431         return None
1432
1433 def p_assert_statement(s):
1434     # s.sy == 'assert'
1435     pos = s.position()
1436     s.next()
1437     cond = p_test(s)
1438     if s.sy == ',':
1439         s.next()
1440         value = p_test(s)
1441     else:
1442         value = None
1443     return Nodes.AssertStatNode(pos, cond = cond, value = value)
1444
1445 statement_terminators = cython.declare(set, set([';', 'NEWLINE', 'EOF']))
1446
1447 def p_if_statement(s):
1448     # s.sy == 'if'
1449     pos = s.position()
1450     s.next()
1451     if_clauses = [p_if_clause(s)]
1452     while s.sy == 'elif':
1453         s.next()
1454         if_clauses.append(p_if_clause(s))
1455     else_clause = p_else_clause(s)
1456     return Nodes.IfStatNode(pos,
1457         if_clauses = if_clauses, else_clause = else_clause)
1458
1459 def p_if_clause(s):
1460     pos = s.position()
1461     test = p_test(s)
1462     body = p_suite(s)
1463     return Nodes.IfClauseNode(pos,
1464         condition = test, body = body)
1465
1466 def p_else_clause(s):
1467     if s.sy == 'else':
1468         s.next()
1469         return p_suite(s)
1470     else:
1471         return None
1472
1473 def p_while_statement(s):
1474     # s.sy == 'while'
1475     pos = s.position()
1476     s.next()
1477     test = p_test(s)
1478     body = p_suite(s)
1479     else_clause = p_else_clause(s)
1480     return Nodes.WhileStatNode(pos,
1481         condition = test, body = body,
1482         else_clause = else_clause)
1483
1484 def p_for_statement(s):
1485     # s.sy == 'for'
1486     pos = s.position()
1487     s.next()
1488     kw = p_for_bounds(s, allow_testlist=True)
1489     body = p_suite(s)
1490     else_clause = p_else_clause(s)
1491     kw.update(body = body, else_clause = else_clause)
1492     return Nodes.ForStatNode(pos, **kw)
1493
1494 def p_for_bounds(s, allow_testlist=True):
1495     target = p_for_target(s)
1496     if s.sy == 'in':
1497         s.next()
1498         iterator = p_for_iterator(s, allow_testlist)
1499         return dict( target = target, iterator = iterator )
1500     elif not s.in_python_file:
1501         if s.sy == 'from':
1502             s.next()
1503             bound1 = p_bit_expr(s)
1504         else:
1505             # Support shorter "for a <= x < b" syntax
1506             bound1, target = target, None
1507         rel1 = p_for_from_relation(s)
1508         name2_pos = s.position()
1509         name2 = p_ident(s)
1510         rel2_pos = s.position()
1511         rel2 = p_for_from_relation(s)
1512         bound2 = p_bit_expr(s)
1513         step = p_for_from_step(s)
1514         if target is None:
1515             target = ExprNodes.NameNode(name2_pos, name = name2)
1516         else:
1517             if not target.is_name:
1518                 error(target.pos,
1519                     "Target of for-from statement must be a variable name")
1520             elif name2 != target.name:
1521                 error(name2_pos,
1522                     "Variable name in for-from range does not match target")
1523         if rel1[0] != rel2[0]:
1524             error(rel2_pos,
1525                 "Relation directions in for-from do not match")
1526         return dict(target = target,
1527                     bound1 = bound1,
1528                     relation1 = rel1,
1529                     relation2 = rel2,
1530                     bound2 = bound2,
1531                     step = step,
1532                     )
1533     else:
1534         s.expect('in')
1535         return {}
1536
1537 def p_for_from_relation(s):
1538     if s.sy in inequality_relations:
1539         op = s.sy
1540         s.next()
1541         return op
1542     else:
1543         s.error("Expected one of '<', '<=', '>' '>='")
1544
1545 def p_for_from_step(s):
1546     if s.sy == 'IDENT' and s.systring == 'by':
1547         s.next()
1548         step = p_bit_expr(s)
1549         return step
1550     else:
1551         return None
1552
1553 inequality_relations = cython.declare(set, set(['<', '<=', '>', '>=']))
1554
1555 def p_target(s, terminator):
1556     pos = s.position()
1557     expr = p_starred_expr(s)
1558     if s.sy == ',':
1559         s.next()
1560         exprs = [expr]
1561         while s.sy != terminator:
1562             exprs.append(p_starred_expr(s))
1563             if s.sy != ',':
1564                 break
1565             s.next()
1566         return ExprNodes.TupleNode(pos, args = exprs)
1567     else:
1568         return expr
1569
1570 def p_for_target(s):
1571     return p_target(s, 'in')
1572
1573 def p_for_iterator(s, allow_testlist=True):
1574     pos = s.position()
1575     if allow_testlist:
1576         expr = p_testlist(s)
1577     else:
1578         expr = p_or_test(s)
1579     return ExprNodes.IteratorNode(pos, sequence = expr)
1580
1581 def p_try_statement(s):
1582     # s.sy == 'try'
1583     pos = s.position()
1584     s.next()
1585     body = p_suite(s)
1586     except_clauses = []
1587     else_clause = None
1588     if s.sy in ('except', 'else'):
1589         while s.sy == 'except':
1590             except_clauses.append(p_except_clause(s))
1591         if s.sy == 'else':
1592             s.next()
1593             else_clause = p_suite(s)
1594         body = Nodes.TryExceptStatNode(pos,
1595             body = body, except_clauses = except_clauses,
1596             else_clause = else_clause)
1597         if s.sy != 'finally':
1598             return body
1599         # try-except-finally is equivalent to nested try-except/try-finally
1600     if s.sy == 'finally':
1601         s.next()
1602         finally_clause = p_suite(s)
1603         return Nodes.TryFinallyStatNode(pos,
1604             body = body, finally_clause = finally_clause)
1605     else:
1606         s.error("Expected 'except' or 'finally'")
1607
1608 def p_except_clause(s):
1609     # s.sy == 'except'
1610     pos = s.position()
1611     s.next()
1612     exc_type = None
1613     exc_value = None
1614     is_except_as = False
1615     if s.sy != ':':
1616         exc_type = p_test(s)
1617         # normalise into list of single exception tests
1618         if isinstance(exc_type, ExprNodes.TupleNode):
1619             exc_type = exc_type.args
1620         else:
1621             exc_type = [exc_type]
1622         if s.sy == ',' or (s.sy == 'IDENT' and s.systring == 'as'
1623                            and s.context.language_level == 2):
1624             s.next()
1625             exc_value = p_test(s)
1626         elif s.sy == 'IDENT' and s.systring == 'as':
1627             # Py3 syntax requires a name here
1628             s.next()
1629             pos2 = s.position()
1630             name = p_ident(s)
1631             exc_value = ExprNodes.NameNode(pos2, name = name)
1632             is_except_as = True
1633     body = p_suite(s)
1634     return Nodes.ExceptClauseNode(pos,
1635         pattern = exc_type, target = exc_value,
1636         body = body, is_except_as=is_except_as)
1637
1638 def p_include_statement(s, ctx):
1639     pos = s.position()
1640     s.next() # 'include'
1641     unicode_include_file_name = p_string_literal(s, 'u')[2]
1642     s.expect_newline("Syntax error in include statement")
1643     if s.compile_time_eval:
1644         include_file_name = unicode_include_file_name
1645         include_file_path = s.context.find_include_file(include_file_name, pos)
1646         if include_file_path:
1647             s.included_files.append(include_file_name)
1648             f = Utils.open_source_file(include_file_path, mode="rU")
1649             source_desc = FileSourceDescriptor(include_file_path)
1650             s2 = PyrexScanner(f, source_desc, s, source_encoding=f.encoding, parse_comments=s.parse_comments)
1651             try:
1652                 tree = p_statement_list(s2, ctx)
1653             finally:
1654                 f.close()
1655             return tree
1656         else:
1657             return None
1658     else:
1659         return Nodes.PassStatNode(pos)
1660
1661 def p_with_statement(s):
1662     s.next() # 'with'
1663     if s.systring == 'template' and not s.in_python_file:
1664         node = p_with_template(s)
1665     else:
1666         node = p_with_items(s)
1667     return node
1668
1669 def p_with_items(s):
1670     pos = s.position()
1671     if not s.in_python_file and s.sy == 'IDENT' and s.systring in ('nogil', 'gil'):
1672         state = s.systring
1673         s.next()
1674         if s.sy == ',':
1675             s.next()
1676             body = p_with_items(s)
1677         else:
1678             body = p_suite(s)
1679         return Nodes.GILStatNode(pos, state = state, body = body)
1680     else:
1681         manager = p_test(s)
1682         target = None
1683         if s.sy == 'IDENT' and s.systring == 'as':
1684             s.next()
1685             target = p_starred_expr(s)
1686         if s.sy == ',':
1687             s.next()
1688             body = p_with_items(s)
1689         else:
1690             body = p_suite(s)
1691     return Nodes.WithStatNode(pos, manager = manager,
1692                               target = target, body = body)
1693
1694 def p_with_template(s):
1695     pos = s.position()
1696     templates = []
1697     s.next()
1698     s.expect('[')
1699     templates.append(s.systring)
1700     s.next()
1701     while s.systring == ',':
1702         s.next()
1703         templates.append(s.systring)
1704         s.next()
1705     s.expect(']')
1706     if s.sy == ':':
1707         s.next()
1708         s.expect_newline("Syntax error in template function declaration")
1709         s.expect_indent()
1710         body_ctx = Ctx()
1711         body_ctx.templates = templates
1712         func_or_var = p_c_func_or_var_declaration(s, pos, body_ctx)
1713         s.expect_dedent()
1714         return func_or_var
1715     else:
1716         error(pos, "Syntax error in template function declaration")
1717
1718 def p_simple_statement(s, first_statement = 0):
1719     #print "p_simple_statement:", s.sy, s.systring ###
1720     if s.sy == 'global':
1721         node = p_global_statement(s)
1722     elif s.sy == 'nonlocal':
1723         node = p_nonlocal_statement(s)
1724     elif s.sy == 'print':
1725         node = p_print_statement(s)
1726     elif s.sy == 'exec':
1727         node = p_exec_statement(s)
1728     elif s.sy == 'del':
1729         node = p_del_statement(s)
1730     elif s.sy == 'break':
1731         node = p_break_statement(s)
1732     elif s.sy == 'continue':
1733         node = p_continue_statement(s)
1734     elif s.sy == 'return':
1735         node = p_return_statement(s)
1736     elif s.sy == 'raise':
1737         node = p_raise_statement(s)
1738     elif s.sy in ('import', 'cimport'):
1739         node = p_import_statement(s)
1740     elif s.sy == 'from':
1741         node = p_from_import_statement(s, first_statement = first_statement)
1742     elif s.sy == 'yield':
1743         node = p_yield_statement(s)
1744     elif s.sy == 'assert':
1745         node = p_assert_statement(s)
1746     elif s.sy == 'pass':
1747         node = p_pass_statement(s)
1748     else:
1749         node = p_expression_or_assignment(s)
1750     return node
1751
1752 def p_simple_statement_list(s, ctx, first_statement = 0):
1753     # Parse a series of simple statements on one line
1754     # separated by semicolons.
1755     stat = p_simple_statement(s, first_statement = first_statement)
1756     pos = stat.pos
1757     stats = []
1758     if not isinstance(stat, Nodes.PassStatNode):
1759         stats.append(stat)
1760     while s.sy == ';':
1761         #print "p_simple_statement_list: maybe more to follow" ###
1762         s.next()
1763         if s.sy in ('NEWLINE', 'EOF'):
1764             break
1765         stat = p_simple_statement(s, first_statement = first_statement)
1766         if isinstance(stat, Nodes.PassStatNode):
1767             continue
1768         stats.append(stat)
1769         first_statement = False
1770
1771     if not stats:
1772         stat = Nodes.PassStatNode(pos)
1773     elif len(stats) == 1:
1774         stat = stats[0]
1775     else:
1776         stat = Nodes.StatListNode(pos, stats = stats)
1777     s.expect_newline("Syntax error in simple statement list")
1778     return stat
1779
1780 def p_compile_time_expr(s):
1781     old = s.compile_time_expr
1782     s.compile_time_expr = 1
1783     expr = p_testlist(s)
1784     s.compile_time_expr = old
1785     return expr
1786
1787 def p_DEF_statement(s):
1788     pos = s.position()
1789     denv = s.compile_time_env
1790     s.next() # 'DEF'
1791     name = p_ident(s)
1792     s.expect('=')
1793     expr = p_compile_time_expr(s)
1794     value = expr.compile_time_value(denv)
1795     #print "p_DEF_statement: %s = %r" % (name, value) ###
1796     denv.declare(name, value)
1797     s.expect_newline()
1798     return Nodes.PassStatNode(pos)
1799
1800 def p_IF_statement(s, ctx):
1801     pos = s.position()
1802     saved_eval = s.compile_time_eval
1803     current_eval = saved_eval
1804     denv = s.compile_time_env
1805     result = None
1806     while 1:
1807         s.next() # 'IF' or 'ELIF'
1808         expr = p_compile_time_expr(s)
1809         s.compile_time_eval = current_eval and bool(expr.compile_time_value(denv))
1810         body = p_suite(s, ctx)
1811         if s.compile_time_eval:
1812             result = body
1813             current_eval = 0
1814         if s.sy != 'ELIF':
1815             break
1816     if s.sy == 'ELSE':
1817         s.next()
1818         s.compile_time_eval = current_eval
1819         body = p_suite(s, ctx)
1820         if current_eval:
1821             result = body
1822     if not result:
1823         result = Nodes.PassStatNode(pos)
1824     s.compile_time_eval = saved_eval
1825     return result
1826
1827 def p_statement(s, ctx, first_statement = 0):
1828     cdef_flag = ctx.cdef_flag
1829     decorators = None
1830     if s.sy == 'ctypedef':
1831         if ctx.level not in ('module', 'module_pxd'):
1832             s.error("ctypedef statement not allowed here")
1833         #if ctx.api:
1834         #    error(s.position(), "'api' not allowed with 'ctypedef'")
1835         return p_ctypedef_statement(s, ctx)
1836     elif s.sy == 'DEF':
1837         return p_DEF_statement(s)
1838     elif s.sy == 'IF':
1839         return p_IF_statement(s, ctx)
1840     elif s.sy == 'DECORATOR':
1841         if ctx.level not in ('module', 'class', 'c_class', 'function', 'property', 'module_pxd', 'c_class_pxd', 'other'):
1842             s.error('decorator not allowed here')
1843         s.level = ctx.level
1844         decorators = p_decorators(s)
1845         bad_toks =  'def', 'cdef', 'cpdef', 'class'
1846         if not ctx.allow_struct_enum_decorator and s.sy not in bad_toks:
1847             s.error("Decorators can only be followed by functions or classes")
1848     elif s.sy == 'pass' and cdef_flag:
1849         # empty cdef block
1850         return p_pass_statement(s, with_newline = 1)
1851
1852     overridable = 0
1853     if s.sy == 'cdef':
1854         cdef_flag = 1
1855         s.next()
1856     elif s.sy == 'cpdef':
1857         cdef_flag = 1
1858         overridable = 1
1859         s.next()
1860     if cdef_flag:
1861         if ctx.level not in ('module', 'module_pxd', 'function', 'c_class', 'c_class_pxd'):
1862             s.error('cdef statement not allowed here')
1863         s.level = ctx.level
1864         node = p_cdef_statement(s, ctx(overridable = overridable))
1865         if decorators is not None:
1866             tup = Nodes.CFuncDefNode, Nodes.CVarDefNode, Nodes.CClassDefNode
1867             if ctx.allow_struct_enum_decorator:
1868                 tup += Nodes.CStructOrUnionDefNode, Nodes.CEnumDefNode
1869             if not isinstance(node, tup):
1870                 s.error("Decorators can only be followed by functions or classes")
1871             node.decorators = decorators
1872         return node
1873     else:
1874         if ctx.api:
1875             s.error("'api' not allowed with this statement", fatal=False)
1876         elif s.sy == 'def':
1877             # def statements aren't allowed in pxd files, except
1878             # as part of a cdef class
1879             if ('pxd' in ctx.level) and (ctx.level != 'c_class_pxd'):
1880                 s.error('def statement not allowed here')
1881             s.level = ctx.level
1882             return p_def_statement(s, decorators)
1883         elif s.sy == 'class':
1884             if ctx.level not in ('module', 'function', 'class', 'other'):
1885                 s.error("class definition not allowed here")
1886             return p_class_statement(s, decorators)
1887         elif s.sy == 'include':
1888             if ctx.level not in ('module', 'module_pxd'):
1889                 s.error("include statement not allowed here")
1890             return p_include_statement(s, ctx)
1891         elif ctx.level == 'c_class' and s.sy == 'IDENT' and s.systring == 'property':
1892             return p_property_decl(s)
1893         elif s.sy == 'pass' and ctx.level != 'property':
1894             return p_pass_statement(s, with_newline=True)
1895         else:
1896             if ctx.level in ('c_class_pxd', 'property'):
1897                 node = p_ignorable_statement(s)
1898                 if node is not None:
1899                     return node
1900                 s.error("Executable statement not allowed here")
1901             if s.sy == 'if':
1902                 return p_if_statement(s)
1903             elif s.sy == 'while':
1904                 return p_while_statement(s)
1905             elif s.sy == 'for':
1906                 return p_for_statement(s)
1907             elif s.sy == 'try':
1908                 return p_try_statement(s)
1909             elif s.sy == 'with':
1910                 return p_with_statement(s)
1911             else:
1912                 return p_simple_statement_list(
1913                     s, ctx, first_statement = first_statement)
1914
1915 def p_statement_list(s, ctx, first_statement = 0):
1916     # Parse a series of statements separated by newlines.
1917     pos = s.position()
1918     stats = []
1919     while s.sy not in ('DEDENT', 'EOF'):
1920         stat = p_statement(s, ctx, first_statement = first_statement)
1921         if isinstance(stat, Nodes.PassStatNode):
1922             continue
1923         stats.append(stat)
1924         first_statement = False
1925     if not stats:
1926         return Nodes.PassStatNode(pos)
1927     elif len(stats) == 1:
1928         return stats[0]
1929     else:
1930         return Nodes.StatListNode(pos, stats = stats)
1931
1932
1933 def p_suite(s, ctx=Ctx()):
1934     return p_suite_with_docstring(s, ctx, with_doc_only=False)[1]
1935
1936
1937 def p_suite_with_docstring(s, ctx, with_doc_only=False):
1938     s.expect(':')
1939     doc = None
1940     if s.sy == 'NEWLINE':
1941         s.next()
1942         s.expect_indent()
1943         if with_doc_only:
1944             doc = p_doc_string(s)
1945         body = p_statement_list(s, ctx)
1946         s.expect_dedent()
1947     else:
1948         if ctx.api:
1949             s.error("'api' not allowed with this statement", fatal=False)
1950         if ctx.level in ('module', 'class', 'function', 'other'):
1951             body = p_simple_statement_list(s, ctx)
1952         else:
1953             body = p_pass_statement(s)
1954             s.expect_newline("Syntax error in declarations")
1955     if not with_doc_only:
1956         doc, body = _extract_docstring(body)
1957     return doc, body
1958
1959
1960 def p_positional_and_keyword_args(s, end_sy_set, templates = None):
1961     """
1962     Parses positional and keyword arguments. end_sy_set
1963     should contain any s.sy that terminate the argument list.
1964     Argument expansion (* and **) are not allowed.
1965
1966     Returns: (positional_args, keyword_args)
1967     """
1968     positional_args = []
1969     keyword_args = []
1970     pos_idx = 0
1971
1972     while s.sy not in end_sy_set:
1973         if s.sy == '*' or s.sy == '**':
1974             s.error('Argument expansion not allowed here.', fatal=False)
1975
1976         parsed_type = False
1977         if s.sy == 'IDENT' and s.peek()[0] == '=':
1978             ident = s.systring
1979             s.next() # s.sy is '='
1980             s.next()
1981             if looking_at_expr(s):
1982                 arg = p_test(s)
1983             else:
1984                 base_type = p_c_base_type(s, templates = templates)
1985                 declarator = p_c_declarator(s, empty = 1)
1986                 arg = Nodes.CComplexBaseTypeNode(base_type.pos,
1987                     base_type = base_type, declarator = declarator)
1988                 parsed_type = True
1989             keyword_node = ExprNodes.IdentifierStringNode(
1990                 arg.pos, value = EncodedString(ident))
1991             keyword_args.append((keyword_node, arg))
1992             was_keyword = True
1993
1994         else:
1995             if looking_at_expr(s):
1996                 arg = p_test(s)
1997             else:
1998                 base_type = p_c_base_type(s, templates = templates)
1999                 declarator = p_c_declarator(s, empty = 1)
2000                 arg = Nodes.CComplexBaseTypeNode(base_type.pos,
2001                     base_type = base_type, declarator = declarator)
2002                 parsed_type = True
2003             positional_args.append(arg)
2004             pos_idx += 1
2005             if len(keyword_args) > 0:
2006                 s.error("Non-keyword arg following keyword arg",
2007                         pos=arg.pos)
2008
2009         if s.sy != ',':
2010             if s.sy not in end_sy_set:
2011                 if parsed_type:
2012                     s.error("Unmatched %s" % " or ".join(end_sy_set))
2013             break
2014         s.next()
2015     return positional_args, keyword_args
2016
2017 def p_c_base_type(s, self_flag = 0, nonempty = 0, templates = None):
2018     # If self_flag is true, this is the base type for the
2019     # self argument of a C method of an extension type.
2020     if s.sy == '(':
2021         return p_c_complex_base_type(s, templates = templates)
2022     else:
2023         return p_c_simple_base_type(s, self_flag, nonempty = nonempty, templates = templates)
2024
2025 def p_calling_convention(s):
2026     if s.sy == 'IDENT' and s.systring in calling_convention_words:
2027         result = s.systring
2028         s.next()
2029         return result
2030     else:
2031         return ""
2032
2033 calling_convention_words = cython.declare(
2034     set, set(["__stdcall", "__cdecl", "__fastcall"]))
2035
2036 def p_c_complex_base_type(s, templates = None):
2037     # s.sy == '('
2038     pos = s.position()
2039     s.next()
2040     base_type = p_c_base_type(s, templates = templates)
2041     declarator = p_c_declarator(s, empty = 1)
2042     s.expect(')')
2043     type_node = Nodes.CComplexBaseTypeNode(pos,
2044             base_type = base_type, declarator = declarator)
2045     if s.sy == '[':
2046         if is_memoryviewslice_access(s):
2047             type_node = p_memoryviewslice_access(s, type_node)
2048         else:
2049             type_node = p_buffer_or_template(s, type_node, templates)
2050     return type_node
2051
2052
2053 def p_c_simple_base_type(s, self_flag, nonempty, templates = None):
2054     #print "p_c_simple_base_type: self_flag =", self_flag, nonempty
2055     is_basic = 0
2056     signed = 1
2057     longness = 0
2058     complex = 0
2059     module_path = []
2060     pos = s.position()
2061     if not s.sy == 'IDENT':
2062         error(pos, "Expected an identifier, found '%s'" % s.sy)
2063     if s.systring == 'const':
2064         s.next()
2065         base_type = p_c_base_type(s,
2066             self_flag = self_flag, nonempty = nonempty, templates = templates)
2067         return Nodes.CConstTypeNode(pos, base_type = base_type)
2068     if looking_at_base_type(s):
2069         #print "p_c_simple_base_type: looking_at_base_type at", s.position()
2070         is_basic = 1
2071         if s.sy == 'IDENT' and s.systring in special_basic_c_types:
2072             signed, longness = special_basic_c_types[s.systring]
2073             name = s.systring
2074             s.next()
2075         else:
2076             signed, longness = p_sign_and_longness(s)
2077             if s.sy == 'IDENT' and s.systring in basic_c_type_names:
2078                 name = s.systring
2079                 s.next()
2080             else:
2081                 name = 'int'  # long [int], short [int], long [int] complex, etc.
2082         if s.sy == 'IDENT' and s.systring == 'complex':
2083             complex = 1
2084             s.next()
2085     elif looking_at_dotted_name(s):
2086         #print "p_c_simple_base_type: looking_at_type_name at", s.position()
2087         name = s.systring
2088         s.next()
2089         while s.sy == '.':
2090             module_path.append(name)
2091             s.next()
2092             name = p_ident(s)
2093     else:
2094         name = s.systring
2095         s.next()
2096         if nonempty and s.sy != 'IDENT':
2097             # Make sure this is not a declaration of a variable or function.
2098             if s.sy == '(':
2099                 s.next()
2100                 if (s.sy == '*' or s.sy == '**' or s.sy == '&'
2101                         or (s.sy == 'IDENT' and s.systring in calling_convention_words)):
2102                     s.put_back('(', '(')
2103                 else:
2104                     s.put_back('(', '(')
2105                     s.put_back('IDENT', name)
2106                     name = None
2107             elif s.sy not in ('*', '**', '[', '&'):
2108                 s.put_back('IDENT', name)
2109                 name = None
2110
2111     type_node = Nodes.CSimpleBaseTypeNode(pos,
2112         name = name, module_path = module_path,
2113         is_basic_c_type = is_basic, signed = signed,
2114         complex = complex, longness = longness,
2115         is_self_arg = self_flag, templates = templates)
2116
2117     #    declarations here.
2118     if s.sy == '[':
2119         if is_memoryviewslice_access(s):
2120             type_node = p_memoryviewslice_access(s, type_node)
2121         else:
2122             type_node = p_buffer_or_template(s, type_node, templates)
2123
2124     if s.sy == '.':
2125         s.next()
2126         name = p_ident(s)
2127         type_node = Nodes.CNestedBaseTypeNode(pos, base_type = type_node, name = name)
2128
2129     return type_node
2130
2131 def p_buffer_or_template(s, base_type_node, templates):
2132     # s.sy == '['
2133     pos = s.position()
2134     s.next()
2135     # Note that buffer_positional_options_count=1, so the only positional argument is dtype.
2136     # For templated types, all parameters are types.
2137     positional_args, keyword_args = (
2138         p_positional_and_keyword_args(s, (']',), templates)
2139     )
2140     s.expect(']')
2141
2142     if s.sy == '[':
2143         base_type_node = p_buffer_or_template(s, base_type_node, templates)
2144
2145     keyword_dict = ExprNodes.DictNode(pos,
2146         key_value_pairs = [
2147             ExprNodes.DictItemNode(pos=key.pos, key=key, value=value)
2148             for key, value in keyword_args
2149         ])
2150     result = Nodes.TemplatedTypeNode(pos,
2151         positional_args = positional_args,
2152         keyword_args = keyword_dict,
2153         base_type_node = base_type_node)
2154     return result
2155
2156 def p_bracketed_base_type(s, base_type_node, nonempty, empty):
2157     # s.sy == '['
2158     if empty and not nonempty:
2159         # sizeof-like thing.  Only anonymous C arrays allowed (int[SIZE]).
2160         return base_type_node
2161     elif not empty and nonempty:
2162         # declaration of either memoryview slice or buffer.
2163         if is_memoryviewslice_access(s):
2164             return p_memoryviewslice_access(s, base_type_node)
2165         else:
2166             return p_buffer_or_template(s, base_type_node, None)
2167             # return p_buffer_access(s, base_type_node)
2168     elif not empty and not nonempty:
2169         # only anonymous C arrays and memoryview slice arrays here.  We
2170         # disallow buffer declarations for now, due to ambiguity with anonymous
2171         # C arrays.
2172         if is_memoryviewslice_access(s):
2173             return p_memoryviewslice_access(s, base_type_node)
2174         else:
2175             return base_type_node
2176
2177 def is_memoryviewslice_access(s):
2178     # s.sy == '['
2179     # a memoryview slice declaration is distinguishable from a buffer access
2180     # declaration by the first entry in the bracketed list.  The buffer will
2181     # not have an unnested colon in the first entry; the memoryview slice will.
2182     saved = [(s.sy, s.systring)]
2183     s.next()
2184     retval = False
2185     if s.systring == ':':
2186         retval = True
2187     elif s.sy == 'INT':
2188         saved.append((s.sy, s.systring))
2189         s.next()
2190         if s.sy == ':':
2191             retval = True
2192
2193     for sv in saved[::-1]:
2194         s.put_back(*sv)
2195
2196     return retval
2197
2198 def p_memoryviewslice_access(s, base_type_node):
2199     # s.sy == '['
2200     pos = s.position()
2201     s.next()
2202     subscripts, _ = p_subscript_list(s)
2203     # make sure each entry in subscripts is a slice
2204     for subscript in subscripts:
2205         if len(subscript) < 2:
2206             s.error("An axis specification in memoryview declaration does not have a ':'.")
2207     s.expect(']')
2208     indexes = make_slice_nodes(pos, subscripts)
2209     result = Nodes.MemoryViewSliceTypeNode(pos,
2210             base_type_node = base_type_node,
2211             axes = indexes)
2212     return result
2213
2214 def looking_at_name(s):
2215     return s.sy == 'IDENT' and not s.systring in calling_convention_words
2216
2217 def looking_at_expr(s):
2218     if s.systring in base_type_start_words:
2219         return False
2220     elif s.sy == 'IDENT':
2221         is_type = False
2222         name = s.systring
2223         dotted_path = []
2224         s.next()
2225
2226         while s.sy == '.':
2227             s.next()
2228             dotted_path.append(s.systring)
2229             s.expect('IDENT')
2230
2231         saved = s.sy, s.systring
2232         if s.sy == 'IDENT':
2233             is_type = True
2234         elif s.sy == '*' or s.sy == '**':
2235             s.next()
2236             is_type = s.sy in (')', ']')
2237             s.put_back(*saved)
2238         elif s.sy == '(':
2239             s.next()
2240             is_type = s.sy == '*'
2241             s.put_back(*saved)
2242         elif s.sy == '[':
2243             s.next()
2244             is_type = s.sy == ']'
2245             s.put_back(*saved)
2246
2247         dotted_path.reverse()
2248         for p in dotted_path:
2249             s.put_back('IDENT', p)
2250             s.put_back('.', '.')
2251
2252         s.put_back('IDENT', name)
2253         return not is_type and saved[0]
2254     else:
2255         return True
2256
2257 def looking_at_base_type(s):
2258     #print "looking_at_base_type?", s.sy, s.systring, s.position()
2259     return s.sy == 'IDENT' and s.systring in base_type_start_words
2260
2261 def looking_at_dotted_name(s):
2262     if s.sy == 'IDENT':
2263         name = s.systring
2264         s.next()
2265         result = s.sy == '.'
2266         s.put_back('IDENT', name)
2267         return result
2268     else:
2269         return 0
2270
2271 def looking_at_call(s):
2272     "See if we're looking at a.b.c("
2273     # Don't mess up the original position, so save and restore it.
2274     # Unfortunately there's no good way to handle this, as a subsequent call
2275     # to next() will not advance the position until it reads a new token.
2276     position = s.start_line, s.start_col
2277     result = looking_at_expr(s) == u'('
2278     if not result:
2279         s.start_line, s.start_col = position
2280     return result
2281
2282 basic_c_type_names = cython.declare(
2283     set, set(["void", "char", "int", "float", "double", "bint"]))
2284
2285 special_basic_c_types = cython.declare(dict, {
2286     # name : (signed, longness)
2287     "Py_UNICODE" : (0, 0),
2288     "Py_UCS4"    : (0, 0),
2289     "Py_ssize_t" : (2, 0),
2290     "ssize_t"    : (2, 0),
2291     "size_t"     : (0, 0),
2292     "ptrdiff_t"  : (2, 0),
2293 })
2294
2295 sign_and_longness_words = cython.declare(
2296     set, set(["short", "long", "signed", "unsigned"]))
2297
2298 base_type_start_words = cython.declare(
2299     set,
2300     basic_c_type_names
2301     | sign_and_longness_words
2302     | set(special_basic_c_types))
2303
2304 struct_enum_union = cython.declare(
2305     set, set(["struct", "union", "enum", "packed"]))
2306
2307 def p_sign_and_longness(s):
2308     signed = 1
2309     longness = 0
2310     while s.sy == 'IDENT' and s.systring in sign_and_longness_words:
2311         if s.systring == 'unsigned':
2312             signed = 0
2313         elif s.systring == 'signed':
2314             signed = 2
2315         elif s.systring == 'short':
2316             longness = -1
2317         elif s.systring == 'long':
2318             longness += 1
2319         s.next()
2320     return signed, longness
2321
2322 def p_opt_cname(s):
2323     literal = p_opt_string_literal(s, 'u')
2324     if literal is not None:
2325         cname = EncodedString(literal)
2326         cname.encoding = s.source_encoding
2327     else:
2328         cname = None
2329     return cname
2330
2331 def p_c_declarator(s, ctx = Ctx(), empty = 0, is_type = 0, cmethod_flag = 0,
2332                    assignable = 0, nonempty = 0,
2333                    calling_convention_allowed = 0):
2334     # If empty is true, the declarator must be empty. If nonempty is true,
2335     # the declarator must be nonempty. Otherwise we don't care.
2336     # If cmethod_flag is true, then if this declarator declares
2337     # a function, it's a C method of an extension type.
2338     pos = s.position()
2339     if s.sy == '(':
2340         s.next()
2341         if s.sy == ')' or looking_at_name(s):
2342             base = Nodes.CNameDeclaratorNode(pos, name = EncodedString(u""), cname = None)
2343             result = p_c_func_declarator(s, pos, ctx, base, cmethod_flag)
2344         else:
2345             result = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
2346                                     cmethod_flag = cmethod_flag,
2347                                     nonempty = nonempty,
2348                                     calling_convention_allowed = 1)
2349             s.expect(')')
2350     else:
2351         result = p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
2352                                        assignable, nonempty)
2353     if not calling_convention_allowed and result.calling_convention and s.sy != '(':
2354         error(s.position(), "%s on something that is not a function"
2355             % result.calling_convention)
2356     while s.sy in ('[', '('):
2357         pos = s.position()
2358         if s.sy == '[':
2359             result = p_c_array_declarator(s, result)
2360         else: # sy == '('
2361             s.next()
2362             result = p_c_func_declarator(s, pos, ctx, result, cmethod_flag)
2363         cmethod_flag = 0
2364     return result
2365
2366 def p_c_array_declarator(s, base):
2367     pos = s.position()
2368     s.next() # '['
2369     if s.sy != ']':
2370         dim = p_testlist(s)
2371     else:
2372         dim = None
2373     s.expect(']')
2374     return Nodes.CArrayDeclaratorNode(pos, base = base, dimension = dim)
2375
2376 def p_c_func_declarator(s, pos, ctx, base, cmethod_flag):
2377     #  Opening paren has already been skipped
2378     args = p_c_arg_list(s, ctx, cmethod_flag = cmethod_flag,
2379                         nonempty_declarators = 0)
2380     ellipsis = p_optional_ellipsis(s)
2381     s.expect(')')
2382     nogil = p_nogil(s)
2383     exc_val, exc_check = p_exception_value_clause(s)
2384     with_gil = p_with_gil(s)
2385     return Nodes.CFuncDeclaratorNode(pos,
2386         base = base, args = args, has_varargs = ellipsis,
2387         exception_value = exc_val, exception_check = exc_check,
2388         nogil = nogil or ctx.nogil or with_gil, with_gil = with_gil)
2389
2390 supported_overloaded_operators = cython.declare(set, set([
2391     '+', '-', '*', '/', '%',
2392     '++', '--', '~', '|', '&', '^', '<<', '>>', ',',
2393     '==', '!=', '>=', '>', '<=', '<',
2394     '[]', '()', '!',
2395 ]))
2396
2397 def p_c_simple_declarator(s, ctx, empty, is_type, cmethod_flag,
2398                           assignable, nonempty):
2399     pos = s.position()
2400     calling_convention = p_calling_convention(s)
2401     if s.sy == '*':
2402         s.next()
2403         if s.systring == 'const':
2404             const_pos = s.position()
2405             s.next()
2406             const_base = p_c_declarator(s, ctx, empty = empty,
2407                                        is_type = is_type,
2408                                        cmethod_flag = cmethod_flag,
2409                                        assignable = assignable,
2410                                        nonempty = nonempty)
2411             base = Nodes.CConstDeclaratorNode(const_pos, base = const_base)
2412         else:
2413             base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
2414                                   cmethod_flag = cmethod_flag,
2415                                   assignable = assignable, nonempty = nonempty)
2416         result = Nodes.CPtrDeclaratorNode(pos,
2417             base = base)
2418     elif s.sy == '**': # scanner returns this as a single token
2419         s.next()
2420         base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
2421                               cmethod_flag = cmethod_flag,
2422                               assignable = assignable, nonempty = nonempty)
2423         result = Nodes.CPtrDeclaratorNode(pos,
2424             base = Nodes.CPtrDeclaratorNode(pos,
2425                 base = base))
2426     elif s.sy == '&':
2427         s.next()
2428         base = p_c_declarator(s, ctx, empty = empty, is_type = is_type,
2429                               cmethod_flag = cmethod_flag,
2430                               assignable = assignable, nonempty = nonempty)
2431         result = Nodes.CReferenceDeclaratorNode(pos, base = base)
2432     else:
2433         rhs = None
2434         if s.sy == 'IDENT':
2435             name = EncodedString(s.systring)
2436             if empty:
2437                 error(s.position(), "Declarator should be empty")
2438             s.next()
2439             cname = p_opt_cname(s)
2440             if name != 'operator' and s.sy == '=' and assignable:
2441                 s.next()
2442                 rhs = p_test(s)
2443         else:
2444             if nonempty:
2445                 error(s.position(), "Empty declarator")
2446             name = ""
2447             cname = None
2448         if cname is None and ctx.namespace is not None and nonempty:
2449             cname = ctx.namespace + "::" + name
2450         if name == 'operator' and ctx.visibility == 'extern' and nonempty:
2451             op = s.sy
2452             if [1 for c in op if c in '+-*/<=>!%&|([^~,']:
2453                 s.next()
2454                 # Handle diphthong operators.
2455                 if op == '(':
2456                     s.expect(')')
2457                     op = '()'
2458                 elif op == '[':
2459                     s.expect(']')
2460                     op = '[]'
2461                 elif op in ('-', '+', '|', '&') and s.sy == op:
2462                     op *= 2       # ++, --, ...
2463                     s.next()
2464                 elif s.sy == '=':
2465                     op += s.sy    # +=, -=, ...
2466                     s.next()
2467                 if op not in supported_overloaded_operators:
2468                     s.error("Overloading operator '%s' not yet supported." % op,
2469                             fatal=False)
2470                 name += op
2471         result = Nodes.CNameDeclaratorNode(pos,
2472             name = name, cname = cname, default = rhs)
2473     result.calling_convention = calling_convention
2474     return result
2475
2476 def p_nogil(s):
2477     if s.sy == 'IDENT' and s.systring == 'nogil':
2478         s.next()
2479         return 1
2480     else:
2481         return 0
2482
2483 def p_with_gil(s):
2484     if s.sy == 'with':
2485         s.next()
2486         s.expect_keyword('gil')
2487         return 1
2488     else:
2489         return 0
2490
2491 def p_exception_value_clause(s):
2492     exc_val = None
2493     exc_check = 0
2494     if s.sy == 'except':
2495         s.next()
2496         if s.sy == '*':
2497             exc_check = 1
2498             s.next()
2499         elif s.sy == '+':
2500             exc_check = '+'
2501             s.next()
2502             if s.sy == 'IDENT':
2503                 name = s.systring
2504                 s.next()
2505                 exc_val = p_name(s, name)
2506         else:
2507             if s.sy == '?':
2508                 exc_check = 1
2509                 s.next()
2510             exc_val = p_test(s)
2511     return exc_val, exc_check
2512
2513 c_arg_list_terminators = cython.declare(set, set(['*', '**', '.', ')']))
2514
2515 def p_c_arg_list(s, ctx = Ctx(), in_pyfunc = 0, cmethod_flag = 0,
2516                  nonempty_declarators = 0, kw_only = 0, annotated = 1):
2517     #  Comma-separated list of C argument declarations, possibly empty.
2518     #  May have a trailing comma.
2519     args = []
2520     is_self_arg = cmethod_flag
2521     while s.sy not in c_arg_list_terminators:
2522         args.append(p_c_arg_decl(s, ctx, in_pyfunc, is_self_arg,
2523             nonempty = nonempty_declarators, kw_only = kw_only,
2524             annotated = annotated))
2525         if s.sy != ',':
2526             break
2527         s.next()
2528         is_self_arg = 0
2529     return args
2530
2531 def p_optional_ellipsis(s):
2532     if s.sy == '.':
2533         expect_ellipsis(s)
2534         return 1
2535     else:
2536         return 0
2537
2538 def p_c_arg_decl(s, ctx, in_pyfunc, cmethod_flag = 0, nonempty = 0,
2539                  kw_only = 0, annotated = 1):
2540     pos = s.position()
2541     not_none = or_none = 0
2542     default = None
2543     annotation = None
2544     if s.in_python_file:
2545         # empty type declaration
2546         base_type = Nodes.CSimpleBaseTypeNode(pos,
2547             name = None, module_path = [],
2548             is_basic_c_type = 0, signed = 0,
2549             complex = 0, longness = 0,
2550             is_self_arg = cmethod_flag, templates = None)
2551     else:
2552         base_type = p_c_base_type(s, cmethod_flag, nonempty = nonempty)
2553     declarator = p_c_declarator(s, ctx, nonempty = nonempty)
2554     if s.sy in ('not', 'or') and not s.in_python_file:
2555         kind = s.sy
2556         s.next()
2557         if s.sy == 'IDENT' and s.systring == 'None':
2558             s.next()
2559         else:
2560             s.error("Expected 'None'")
2561         if not in_pyfunc:
2562             error(pos, "'%s None' only allowed in Python functions" % kind)
2563         or_none = kind == 'or'
2564         not_none = kind == 'not'
2565     if annotated and s.sy == ':':
2566         s.next()
2567         annotation = p_test(s)
2568     if s.sy == '=':
2569         s.next()
2570         if 'pxd' in ctx.level:
2571             if s.sy not in ['*', '?']:
2572                 error(pos, "default values cannot be specified in pxd files, use ? or *")
2573             default = ExprNodes.BoolNode(1)
2574             s.next()
2575         else:
2576             default = p_test(s)
2577     return Nodes.CArgDeclNode(pos,
2578         base_type = base_type,
2579         declarator = declarator,
2580         not_none = not_none,
2581         or_none = or_none,
2582         default = default,
2583         annotation = annotation,
2584         kw_only = kw_only)
2585
2586 def p_api(s):
2587     if s.sy == 'IDENT' and s.systring == 'api':
2588         s.next()
2589         return 1
2590     else:
2591         return 0
2592
2593 def p_cdef_statement(s, ctx):
2594     pos = s.position()
2595     ctx.visibility = p_visibility(s, ctx.visibility)
2596     ctx.api = ctx.api or p_api(s)
2597     if ctx.api:
2598         if ctx.visibility not in ('private', 'public'):
2599             error(pos, "Cannot combine 'api' with '%s'" % ctx.visibility)
2600     if (ctx.visibility == 'extern') and s.sy == 'from':
2601         return p_cdef_extern_block(s, pos, ctx)
2602     elif s.sy == 'import':
2603         s.next()
2604         return p_cdef_extern_block(s, pos, ctx)
2605     elif p_nogil(s):
2606         ctx.nogil = 1
2607         if ctx.overridable:
2608             error(pos, "cdef blocks cannot be declared cpdef")
2609         return p_cdef_block(s, ctx)
2610     elif s.sy == ':':
2611         if ctx.overridable:
2612             error(pos, "cdef blocks cannot be declared cpdef")
2613         return p_cdef_block(s, ctx)
2614     elif s.sy == 'class':
2615         if ctx.level not in ('module', 'module_pxd'):
2616             error(pos, "Extension type definition not allowed here")
2617         if ctx.overridable:
2618             error(pos, "Extension types cannot be declared cpdef")
2619         return p_c_class_definition(s, pos, ctx)
2620     elif s.sy == 'IDENT' and s.systring == 'cppclass':
2621         return p_cpp_class_definition(s, pos, ctx)
2622     elif s.sy == 'IDENT' and s.systring in struct_enum_union:
2623         if ctx.level not in ('module', 'module_pxd'):
2624             error(pos, "C struct/union/enum definition not allowed here")
2625         if ctx.overridable:
2626             error(pos, "C struct/union/enum cannot be declared cpdef")
2627         return p_struct_enum(s, pos, ctx)
2628     elif s.sy == 'IDENT' and s.systring == 'fused':
2629         return p_fused_definition(s, pos, ctx)
2630     else:
2631         return p_c_func_or_var_declaration(s, pos, ctx)
2632
2633 def p_cdef_block(s, ctx):
2634     return p_suite(s, ctx(cdef_flag = 1))
2635
2636 def p_cdef_extern_block(s, pos, ctx):
2637     if ctx.overridable:
2638         error(pos, "cdef extern blocks cannot be declared cpdef")
2639     include_file = None
2640     s.expect('from')
2641     if s.sy == '*':
2642         s.next()
2643     else:
2644         include_file = p_string_literal(s, 'u')[2]
2645     ctx = ctx(cdef_flag = 1, visibility = 'extern')
2646     if s.systring == "namespace":
2647         s.next()
2648         ctx.namespace = p_string_literal(s, 'u')[2]
2649     if p_nogil(s):
2650         ctx.nogil = 1
2651     body = p_suite(s, ctx)
2652     return Nodes.CDefExternNode(pos,
2653         include_file = include_file,
2654         body = body,
2655         namespace = ctx.namespace)
2656
2657 def p_c_enum_definition(s, pos, ctx):
2658     # s.sy == ident 'enum'
2659     s.next()
2660     if s.sy == 'IDENT':
2661         name = s.systring
2662         s.next()
2663         cname = p_opt_cname(s)
2664         if cname is None and ctx.namespace is not None:
2665             cname = ctx.namespace + "::" + name
2666     else:
2667         name = None
2668         cname = None
2669     items = None
2670     s.expect(':')
2671     items = []
2672     if s.sy != 'NEWLINE':
2673         p_c_enum_line(s, ctx, items)
2674     else:
2675         s.next() # 'NEWLINE'
2676         s.expect_indent()
2677         while s.sy not in ('DEDENT', 'EOF'):
2678             p_c_enum_line(s, ctx, items)
2679         s.expect_dedent()
2680     return Nodes.CEnumDefNode(
2681         pos, name = name, cname = cname, items = items,
2682         typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
2683         api = ctx.api, in_pxd = ctx.level == 'module_pxd')
2684
2685 def p_c_enum_line(s, ctx, items):
2686     if s.sy != 'pass':
2687         p_c_enum_item(s, ctx, items)
2688         while s.sy == ',':
2689             s.next()
2690             if s.sy in ('NEWLINE', 'EOF'):
2691                 break
2692             p_c_enum_item(s, ctx, items)
2693     else:
2694         s.next()
2695     s.expect_newline("Syntax error in enum item list")
2696
2697 def p_c_enum_item(s, ctx, items):
2698     pos = s.position()
2699     name = p_ident(s)
2700     cname = p_opt_cname(s)
2701     if cname is None and ctx.namespace is not None:
2702         cname = ctx.namespace + "::" + name
2703     value = None
2704     if s.sy == '=':
2705         s.next()
2706         value = p_test(s)
2707     items.append(Nodes.CEnumDefItemNode(pos,
2708         name = name, cname = cname, value = value))
2709
2710 def p_c_struct_or_union_definition(s, pos, ctx):
2711     packed = False
2712     if s.systring == 'packed':
2713         packed = True
2714         s.next()
2715         if s.sy != 'IDENT' or s.systring != 'struct':
2716             s.expected('struct')
2717     # s.sy == ident 'struct' or 'union'
2718     kind = s.systring
2719     s.next()
2720     name = p_ident(s)
2721     cname = p_opt_cname(s)
2722     if cname is None and ctx.namespace is not None:
2723         cname = ctx.namespace + "::" + name
2724     attributes = None
2725     if s.sy == ':':
2726         s.next()
2727         s.expect('NEWLINE')
2728         s.expect_indent()
2729         attributes = []
2730         body_ctx = Ctx()
2731         while s.sy != 'DEDENT':
2732             if s.sy != 'pass':
2733                 attributes.append(
2734                     p_c_func_or_var_declaration(s, s.position(), body_ctx))
2735             else:
2736                 s.next()
2737                 s.expect_newline("Expected a newline")
2738         s.expect_dedent()
2739     else:
2740         s.expect_newline("Syntax error in struct or union definition")
2741     return Nodes.CStructOrUnionDefNode(pos,
2742         name = name, cname = cname, kind = kind, attributes = attributes,
2743         typedef_flag = ctx.typedef_flag, visibility = ctx.visibility,
2744         api = ctx.api, in_pxd = ctx.level == 'module_pxd', packed = packed)
2745
2746 def p_fused_definition(s, pos, ctx):
2747     """
2748     c(type)def fused my_fused_type:
2749         ...
2750     """
2751     # s.systring == 'fused'
2752
2753     if ctx.level not in ('module', 'module_pxd'):
2754         error(pos, "Fused type definition not allowed here")
2755
2756     s.next()
2757     name = p_ident(s)
2758
2759     s.expect(":")
2760     s.expect_newline()
2761     s.expect_indent()
2762
2763     types = []
2764     while s.sy != 'DEDENT':
2765         if s.sy != 'pass':
2766             #types.append(p_c_declarator(s))
2767             types.append(p_c_base_type(s)) #, nonempty=1))
2768         else:
2769             s.next()
2770
2771         s.expect_newline()
2772
2773     s.expect_dedent()
2774
2775     if not types:
2776         error(pos, "Need at least one type")
2777
2778     return Nodes.FusedTypeNode(pos, name=name, types=types)
2779
2780 def p_struct_enum(s, pos, ctx):
2781     if s.systring == 'enum':
2782         return p_c_enum_definition(s, pos, ctx)
2783     else:
2784         return p_c_struct_or_union_definition(s, pos, ctx)
2785
2786 def p_visibility(s, prev_visibility):
2787     pos = s.position()
2788     visibility = prev_visibility
2789     if s.sy == 'IDENT' and s.systring in ('extern', 'public', 'readonly'):
2790         visibility = s.systring
2791         if prev_visibility != 'private' and visibility != prev_visibility:
2792             s.error("Conflicting visibility options '%s' and '%s'"
2793                 % (prev_visibility, visibility), fatal=False)
2794         s.next()
2795     return visibility
2796
2797 def p_c_modifiers(s):
2798     if s.sy == 'IDENT' and s.systring in ('inline',):
2799         modifier = s.systring
2800         s.next()
2801         return [modifier] + p_c_modifiers(s)
2802     return []
2803
2804 def p_c_func_or_var_declaration(s, pos, ctx):
2805     cmethod_flag = ctx.level in ('c_class', 'c_class_pxd')
2806     modifiers = p_c_modifiers(s)
2807     base_type = p_c_base_type(s, nonempty = 1, templates = ctx.templates)
2808     declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
2809                                 assignable = 1, nonempty = 1)
2810     declarator.overridable = ctx.overridable
2811     if s.sy == 'IDENT' and s.systring == 'const' and ctx.level == 'cpp_class':
2812         s.next()
2813         is_const_method = 1
2814     else:
2815         is_const_method = 0
2816     if s.sy == ':':
2817         if ctx.level not in ('module', 'c_class', 'module_pxd', 'c_class_pxd', 'cpp_class') and not ctx.templates:
2818             s.error("C function definition not allowed here")
2819         doc, suite = p_suite_with_docstring(s, Ctx(level='function'))
2820         result = Nodes.CFuncDefNode(pos,
2821             visibility = ctx.visibility,
2822             base_type = base_type,
2823             declarator = declarator,
2824             body = suite,
2825             doc = doc,
2826             modifiers = modifiers,
2827             api = ctx.api,
2828             overridable = ctx.overridable,
2829             is_const_method = is_const_method)
2830     else:
2831         #if api:
2832         #    s.error("'api' not allowed with variable declaration")
2833         if is_const_method:
2834             declarator.is_const_method = is_const_method
2835         declarators = [declarator]
2836         while s.sy == ',':
2837             s.next()
2838             if s.sy == 'NEWLINE':
2839                 break
2840             declarator = p_c_declarator(s, ctx, cmethod_flag = cmethod_flag,
2841                                         assignable = 1, nonempty = 1)
2842             declarators.append(declarator)
2843         doc_line = s.start_line + 1
2844         s.expect_newline("Syntax error in C variable declaration")
2845         if ctx.level in ('c_class', 'c_class_pxd') and s.start_line == doc_line:
2846             doc = p_doc_string(s)
2847         else:
2848             doc = None
2849         result = Nodes.CVarDefNode(pos,
2850             visibility = ctx.visibility,
2851             base_type = base_type,
2852             declarators = declarators,
2853             in_pxd = ctx.level in ('module_pxd', 'c_class_pxd'),
2854             doc = doc,
2855             api = ctx.api,
2856             modifiers = modifiers,
2857             overridable = ctx.overridable)
2858     return result
2859
2860 def p_ctypedef_statement(s, ctx):
2861     # s.sy == 'ctypedef'
2862     pos = s.position()
2863     s.next()
2864     visibility = p_visibility(s, ctx.visibility)
2865     api = p_api(s)
2866     ctx = ctx(typedef_flag = 1, visibility = visibility)
2867     if api:
2868         ctx.api = 1
2869     if s.sy == 'class':
2870         return p_c_class_definition(s, pos, ctx)
2871     elif s.sy == 'IDENT' and s.systring in struct_enum_union:
2872         return p_struct_enum(s, pos, ctx)
2873     elif s.sy == 'IDENT' and s.systring == 'fused':
2874         return p_fused_definition(s, pos, ctx)
2875     else:
2876         base_type = p_c_base_type(s, nonempty = 1)
2877         declarator = p_c_declarator(s, ctx, is_type = 1, nonempty = 1)
2878         s.expect_newline("Syntax error in ctypedef statement")
2879         return Nodes.CTypeDefNode(
2880             pos, base_type = base_type,
2881             declarator = declarator,
2882             visibility = visibility, api = api,
2883             in_pxd = ctx.level == 'module_pxd')
2884
2885 def p_decorators(s):
2886     decorators = []
2887     while s.sy == 'DECORATOR':
2888         pos = s.position()
2889         s.next()
2890         decstring = p_dotted_name(s, as_allowed=0)[2]
2891         names = decstring.split('.')
2892         decorator = ExprNodes.NameNode(pos, name=EncodedString(names[0]))
2893         for name in names[1:]:
2894             decorator = ExprNodes.AttributeNode(pos,
2895                                            attribute=EncodedString(name),
2896                                            obj=decorator)
2897         if s.sy == '(':
2898             decorator = p_call(s, decorator)
2899         decorators.append(Nodes.DecoratorNode(pos, decorator=decorator))
2900         s.expect_newline("Expected a newline after decorator")
2901     return decorators
2902
2903 def p_def_statement(s, decorators=None):
2904     # s.sy == 'def'
2905     pos = s.position()
2906     s.next()
2907     name = EncodedString( p_ident(s) )
2908     s.expect('(')
2909     args, star_arg, starstar_arg = p_varargslist(s, terminator=')')
2910     s.expect(')')
2911     if p_nogil(s):
2912         error(pos, "Python function cannot be declared nogil")
2913     return_type_annotation = None
2914     if s.sy == '->':
2915         s.next()
2916         return_type_annotation = p_test(s)
2917     doc, body = p_suite_with_docstring(s, Ctx(level='function'))
2918     return Nodes.DefNode(pos, name = name, args = args,
2919         star_arg = star_arg, starstar_arg = starstar_arg,
2920         doc = doc, body = body, decorators = decorators,
2921         return_type_annotation = return_type_annotation)
2922
2923 def p_varargslist(s, terminator=')', annotated=1):
2924     args = p_c_arg_list(s, in_pyfunc = 1, nonempty_declarators = 1,
2925                         annotated = annotated)
2926     star_arg = None
2927     starstar_arg = None
2928     if s.sy == '*':
2929         s.next()
2930         if s.sy == 'IDENT':
2931             star_arg = p_py_arg_decl(s, annotated=annotated)
2932         if s.sy == ',':
2933             s.next()
2934             args.extend(p_c_arg_list(s, in_pyfunc = 1,
2935                 nonempty_declarators = 1, kw_only = 1, annotated = annotated))
2936         elif s.sy != terminator:
2937             s.error("Syntax error in Python function argument list")
2938     if s.sy == '**':
2939         s.next()
2940         starstar_arg = p_py_arg_decl(s, annotated=annotated)
2941     return (args, star_arg, starstar_arg)
2942
2943 def p_py_arg_decl(s, annotated = 1):
2944     pos = s.position()
2945     name = p_ident(s)
2946     annotation = None
2947     if annotated and s.sy == ':':
2948         s.next()
2949         annotation = p_test(s)
2950     return Nodes.PyArgDeclNode(pos, name = name, annotation = annotation)
2951
2952 def p_class_statement(s, decorators):
2953     # s.sy == 'class'
2954     pos = s.position()
2955     s.next()
2956     class_name = EncodedString( p_ident(s) )
2957     class_name.encoding = s.source_encoding
2958     arg_tuple = None
2959     keyword_dict = None
2960     starstar_arg = None
2961     if s.sy == '(':
2962         positional_args, keyword_args, star_arg, starstar_arg = \
2963                             p_call_parse_args(s, allow_genexp = False)
2964         arg_tuple, keyword_dict = p_call_build_packed_args(
2965             pos, positional_args, keyword_args, star_arg, None)
2966     if arg_tuple is None:
2967         # XXX: empty arg_tuple
2968         arg_tuple = ExprNodes.TupleNode(pos, args=[])
2969     doc, body = p_suite_with_docstring(s, Ctx(level='class'))
2970     return Nodes.PyClassDefNode(
2971         pos, name=class_name,
2972         bases=arg_tuple,
2973         keyword_args=keyword_dict,
2974         starstar_arg=starstar_arg,
2975         doc=doc, body=body, decorators=decorators,
2976         force_py3_semantics=s.context.language_level >= 3)
2977
2978 def p_c_class_definition(s, pos,  ctx):
2979     # s.sy == 'class'
2980     s.next()
2981     module_path = []
2982     class_name = p_ident(s)
2983     while s.sy == '.':
2984         s.next()
2985         module_path.append(class_name)
2986         class_name = p_ident(s)
2987     if module_path and ctx.visibility != 'extern':
2988         error(pos, "Qualified class name only allowed for 'extern' C class")
2989     if module_path and s.sy == 'IDENT' and s.systring == 'as':
2990         s.next()
2991         as_name = p_ident(s)
2992     else:
2993         as_name = class_name
2994     objstruct_name = None
2995     typeobj_name = None
2996     base_class_module = None
2997     base_class_name = None
2998     if s.sy == '(':
2999         s.next()
3000         base_class_path = [p_ident(s)]
3001         while s.sy == '.':
3002             s.next()
3003             base_class_path.append(p_ident(s))
3004         if s.sy == ',':
3005             s.error("C class may only have one base class", fatal=False)
3006         s.expect(')')
3007         base_class_module = ".".join(base_class_path[:-1])
3008         base_class_name = base_class_path[-1]
3009     if s.sy == '[':
3010         if ctx.visibility not in ('public', 'extern') and not ctx.api:
3011             error(s.position(), "Name options only allowed for 'public', 'api', or 'extern' C class")
3012         objstruct_name, typeobj_name = p_c_class_options(s)
3013     if s.sy == ':':
3014         if ctx.level == 'module_pxd':
3015             body_level = 'c_class_pxd'
3016         else:
3017             body_level = 'c_class'
3018         doc, body = p_suite_with_docstring(s, Ctx(level=body_level))
3019     else:
3020         s.expect_newline("Syntax error in C class definition")
3021         doc = None
3022         body = None
3023     if ctx.visibility == 'extern':
3024         if not module_path:
3025             error(pos, "Module name required for 'extern' C class")
3026         if typeobj_name:
3027             error(pos, "Type object name specification not allowed for 'extern' C class")
3028     elif ctx.visibility == 'public':
3029         if not objstruct_name:
3030             error(pos, "Object struct name specification required for 'public' C class")
3031         if not typeobj_name:
3032             error(pos, "Type object name specification required for 'public' C class")
3033     elif ctx.visibility == 'private':
3034         if ctx.api:
3035             if not objstruct_name:
3036                 error(pos, "Object struct name specification required for 'api' C class")
3037             if not typeobj_name:
3038                 error(pos, "Type object name specification required for 'api' C class")
3039     else:
3040         error(pos, "Invalid class visibility '%s'" % ctx.visibility)
3041     return Nodes.CClassDefNode(pos,
3042         visibility = ctx.visibility,
3043         typedef_flag = ctx.typedef_flag,
3044         api = ctx.api,
3045         module_name = ".".join(module_path),
3046         class_name = class_name,
3047         as_name = as_name,
3048         base_class_module = base_class_module,
3049         base_class_name = base_class_name,
3050         objstruct_name = objstruct_name,
3051         typeobj_name = typeobj_name,
3052         in_pxd = ctx.level == 'module_pxd',
3053         doc = doc,
3054         body = body)
3055
3056 def p_c_class_options(s):
3057     objstruct_name = None
3058     typeobj_name = None
3059     s.expect('[')
3060     while 1:
3061         if s.sy != 'IDENT':
3062             break
3063         if s.systring == 'object':
3064             s.next()
3065             objstruct_name = p_ident(s)
3066         elif s.systring == 'type':
3067             s.next()
3068             typeobj_name = p_ident(s)
3069         if s.sy != ',':
3070             break
3071         s.next()
3072     s.expect(']', "Expected 'object' or 'type'")
3073     return objstruct_name, typeobj_name
3074
3075
3076 def p_property_decl(s):
3077     pos = s.position()
3078     s.next()  # 'property'
3079     name = p_ident(s)
3080     doc, body = p_suite_with_docstring(
3081         s, Ctx(level='property'), with_doc_only=True)
3082     return Nodes.PropertyNode(pos, name=name, doc=doc, body=body)
3083
3084
3085 def p_ignorable_statement(s):
3086     """
3087     Parses any kind of ignorable statement that is allowed in .pxd files.
3088     """
3089     if s.sy == 'BEGIN_STRING':
3090         pos = s.position()
3091         string_node = p_atom(s)
3092         if s.sy != 'EOF':
3093             s.expect_newline("Syntax error in string")
3094         return Nodes.ExprStatNode(pos, expr=string_node)
3095     return None
3096
3097
3098 def p_doc_string(s):
3099     if s.sy == 'BEGIN_STRING':
3100         pos = s.position()
3101         kind, bytes_result, unicode_result = p_cat_string_literal(s)
3102         if s.sy != 'EOF':
3103             s.expect_newline("Syntax error in doc string")
3104         if kind in ('u', ''):
3105             return unicode_result
3106         warning(pos, "Python 3 requires docstrings to be unicode strings")
3107         return bytes_result
3108     else:
3109         return None
3110
3111
3112 def _extract_docstring(node):
3113     """
3114     Extract a docstring from a statement or from the first statement
3115     in a list.  Remove the statement if found.  Return a tuple
3116     (plain-docstring or None, node).
3117     """
3118     doc_node = None
3119     if node is None:
3120         pass
3121     elif isinstance(node, Nodes.ExprStatNode):
3122         if node.expr.is_string_literal:
3123             doc_node = node.expr
3124             node = Nodes.StatListNode(node.pos, stats=[])
3125     elif isinstance(node, Nodes.StatListNode) and node.stats:
3126         stats = node.stats
3127         if isinstance(stats[0], Nodes.ExprStatNode):
3128             if stats[0].expr.is_string_literal:
3129                 doc_node = stats[0].expr
3130                 del stats[0]
3131
3132     if doc_node is None:
3133         doc = None
3134     elif isinstance(doc_node, ExprNodes.BytesNode):
3135         warning(node.pos,
3136                 "Python 3 requires docstrings to be unicode strings")
3137         doc = doc_node.value
3138     elif isinstance(doc_node, ExprNodes.StringNode):
3139         doc = doc_node.unicode_value
3140         if doc is None:
3141             doc = doc_node.value
3142     else:
3143         doc = doc_node.value
3144     return doc, node
3145
3146
3147 def p_code(s, level=None, ctx=Ctx):
3148     body = p_statement_list(s, ctx(level = level), first_statement = 1)
3149     if s.sy != 'EOF':
3150         s.error("Syntax error in statement [%s,%s]" % (
3151             repr(s.sy), repr(s.systring)))
3152     return body
3153
3154 _match_compiler_directive_comment = cython.declare(object, re.compile(
3155     r"^#\s*cython\s*:\s*((\w|[.])+\s*=.*)$").match)
3156
3157 def p_compiler_directive_comments(s):
3158     result = {}
3159     while s.sy == 'commentline':
3160         m = _match_compiler_directive_comment(s.systring)
3161         if m:
3162             directives = m.group(1).strip()
3163             try:
3164                 result.update(Options.parse_directive_list(
3165                     directives, ignore_unknown=True))
3166             except ValueError, e:
3167                 s.error(e.args[0], fatal=False)
3168         s.next()
3169     return result
3170
3171 def p_module(s, pxd, full_module_name, ctx=Ctx):
3172     pos = s.position()
3173
3174     directive_comments = p_compiler_directive_comments(s)
3175     s.parse_comments = False
3176
3177     if 'language_level' in directive_comments:
3178         s.context.set_language_level(directive_comments['language_level'])
3179
3180     doc = p_doc_string(s)
3181     if pxd:
3182         level = 'module_pxd'
3183     else:
3184         level = 'module'
3185
3186     body = p_statement_list(s, ctx(level=level), first_statement = 1)
3187     if s.sy != 'EOF':
3188         s.error("Syntax error in statement [%s,%s]" % (
3189             repr(s.sy), repr(s.systring)))
3190     return ModuleNode(pos, doc = doc, body = body,
3191                       full_module_name = full_module_name,
3192                       directive_comments = directive_comments)
3193
3194 def p_cpp_class_definition(s, pos,  ctx):
3195     # s.sy == 'cppclass'
3196     s.next()
3197     module_path = []
3198     class_name = p_ident(s)
3199     cname = p_opt_cname(s)
3200     if cname is None and ctx.namespace is not None:
3201         cname = ctx.namespace + "::" + class_name
3202     if s.sy == '.':
3203         error(pos, "Qualified class name not allowed C++ class")
3204     if s.sy == '[':
3205         s.next()
3206         templates = [p_ident(s)]
3207         while s.sy == ',':
3208             s.next()
3209             templates.append(p_ident(s))
3210         s.expect(']')
3211     else:
3212         templates = None
3213     if s.sy == '(':
3214         s.next()
3215         base_classes = [p_c_base_type(s, templates = templates)]
3216         while s.sy == ',':
3217             s.next()
3218             base_classes.append(p_c_base_type(s, templates = templates))
3219         s.expect(')')
3220     else:
3221         base_classes = []
3222     if s.sy == '[':
3223         error(s.position(), "Name options not allowed for C++ class")
3224     nogil = p_nogil(s)
3225     if s.sy == ':':
3226         s.next()
3227         s.expect('NEWLINE')
3228         s.expect_indent()
3229         attributes = []
3230         body_ctx = Ctx(visibility = ctx.visibility, level='cpp_class', nogil=nogil or ctx.nogil)
3231         body_ctx.templates = templates
3232         while s.sy != 'DEDENT':
3233             if s.systring == 'cppclass':
3234                 attributes.append(
3235                     p_cpp_class_definition(s, s.position(), body_ctx))
3236             elif s.sy != 'pass':
3237                 attributes.append(
3238                     p_c_func_or_var_declaration(s, s.position(), body_ctx))
3239             else:
3240                 s.next()
3241                 s.expect_newline("Expected a newline")
3242         s.expect_dedent()
3243     else:
3244         attributes = None
3245         s.expect_newline("Syntax error in C++ class definition")
3246     return Nodes.CppClassNode(pos,
3247         name = class_name,
3248         cname = cname,
3249         base_classes = base_classes,
3250         visibility = ctx.visibility,
3251         in_pxd = ctx.level == 'module_pxd',
3252         attributes = attributes,
3253         templates = templates)
3254
3255
3256
3257 #----------------------------------------------
3258 #
3259 #   Debugging
3260 #
3261 #----------------------------------------------
3262
3263 def print_parse_tree(f, node, level, key = None):
3264     from types import ListType, TupleType
3265     from Nodes import Node
3266     ind = "  " * level
3267     if node:
3268         f.write(ind)
3269         if key:
3270             f.write("%s: " % key)
3271         t = type(node)
3272         if t is tuple:
3273             f.write("(%s @ %s\n" % (node[0], node[1]))
3274             for i in xrange(2, len(node)):
3275                 print_parse_tree(f, node[i], level+1)
3276             f.write("%s)\n" % ind)
3277             return
3278         elif isinstance(node, Node):
3279             try:
3280                 tag = node.tag
3281             except AttributeError:
3282                 tag = node.__class__.__name__
3283             f.write("%s @ %s\n" % (tag, node.pos))
3284             for name, value in node.__dict__.items():
3285                 if name != 'tag' and name != 'pos':
3286                     print_parse_tree(f, value, level+1, name)
3287             return
3288         elif t is list:
3289             f.write("[\n")
3290             for i in xrange(len(node)):
3291                 print_parse_tree(f, node[i], level+1)
3292             f.write("%s]\n" % ind)
3293             return
3294     f.write("%s%s\n" % (ind, node))