6 from cStringIO
import StringIO
8 from compiler
import ast
, parse
, walk
9 from compiler
import pyassem
, misc
10 from compiler
.pyassem
import CO_VARARGS
, CO_VARKEYWORDS
, CO_NEWLOCALS
, TupleArg
12 callfunc_opcode_info
= {
13 # (Have *args, Have **args) : opcode
14 (0,0) : "CALL_FUNCTION",
15 (1,0) : "CALL_FUNCTION_VAR",
16 (0,1) : "CALL_FUNCTION_KW",
17 (1,1) : "CALL_FUNCTION_VAR_KW",
20 def compile(filename
):
24 mod
= Module(buf
, filename
)
26 f
= open(filename
+ "c", "wb")
31 def __init__(self
, source
, filename
):
32 self
.filename
= filename
37 ast
= parse(self
.source
)
38 root
, filename
= os
.path
.split(self
.filename
)
39 gen
= ModuleCodeGenerator(filename
)
41 self
.code
= gen
.getCode()
44 f
.write(self
.getPycHeader())
45 marshal
.dump(self
.code
, f
)
47 MAGIC
= (50811 |
(ord('\r')<<16) |
(ord('\n')<<24))
49 def getPycHeader(self
):
50 # compile.c uses marshal to write a long directly, with
51 # calling the interface that would also generate a 1-byte code
52 # to indicate the type of the value. simplest way to get the
53 # same effect is to call marshal and then skip the code.
54 magic
= marshal
.dumps(self
.MAGIC
)[1:]
55 mtime
= os
.stat(self
.filename
)[stat
.ST_MTIME
]
56 mtime
= struct
.pack('i', mtime
)
61 optimized
= 0 # is namespace access optimized?
63 def __init__(self
, filename
):
64 ## Subclasses must define a constructor that intializes self.graph
65 ## before calling this init function
66 ## self.graph = pyassem.PyFlowGraph()
67 self
.filename
= filename
68 self
.locals = misc
.Stack()
69 self
.loops
= misc
.Stack()
72 self
._setupGraphDelegation
()
74 def _setupGraphDelegation(self
):
75 self
.emit
= self
.graph
.emit
76 self
.newBlock
= self
.graph
.newBlock
77 self
.startBlock
= self
.graph
.startBlock
78 self
.nextBlock
= self
.graph
.nextBlock
79 self
.setDocstring
= self
.graph
.setDocstring
82 """Return a code object"""
83 return self
.graph
.getCode()
85 # Next five methods handle name access
87 def isLocalName(self
, name
):
88 return self
.locals.top().has_elt(name
)
90 def storeName(self
, name
):
91 self
._nameOp
('STORE', name
)
93 def loadName(self
, name
):
94 self
._nameOp
('LOAD', name
)
96 def delName(self
, name
):
97 self
._nameOp
('DELETE', name
)
99 def _nameOp(self
, prefix
, name
):
100 if not self
.optimized
:
101 self
.emit(prefix
+ '_NAME', name
)
103 if self
.isLocalName(name
):
104 self
.emit(prefix
+ '_FAST', name
)
106 self
.emit(prefix
+ '_GLOBAL', name
)
108 def set_lineno(self
, node
):
109 """Emit SET_LINENO if node has lineno attribute
111 Returns true if SET_LINENO was emitted.
113 There are no rules for when an AST node should have a lineno
114 attribute. The transformer and AST code need to be reviewed
115 and a consistent policy implemented and documented. Until
116 then, this method works around missing line numbers.
118 lineno
= getattr(node
, 'lineno', None)
119 if lineno
is not None:
120 self
.emit('SET_LINENO', lineno
)
124 # The first few visitor methods handle nodes that generator new
127 def visitModule(self
, node
):
128 lnf
= walk(node
.node
, LocalNameFinder(), 0)
129 self
.locals.push(lnf
.getLocals())
130 self
.setDocstring(node
.doc
)
131 self
.visit(node
.node
)
132 self
.emit('LOAD_CONST', None)
133 self
.emit('RETURN_VALUE')
135 def visitFunction(self
, node
):
136 self
._visitFuncOrLambda
(node
, isLambda
=0)
137 self
.storeName(node
.name
)
139 def visitLambda(self
, node
):
140 self
._visitFuncOrLambda
(node
, isLambda
=1)
141 ## self.storeName("<lambda>")
143 def _visitFuncOrLambda(self
, node
, isLambda
):
144 gen
= FunctionCodeGenerator(node
, self
.filename
, isLambda
)
147 self
.set_lineno(node
)
148 for default
in node
.defaults
:
150 self
.emit('LOAD_CONST', gen
.getCode())
151 self
.emit('MAKE_FUNCTION', len(node
.defaults
))
153 def visitClass(self
, node
):
154 gen
= ClassCodeGenerator(node
, self
.filename
)
157 self
.set_lineno(node
)
158 self
.emit('LOAD_CONST', node
.name
)
159 for base
in node
.bases
:
161 self
.emit('BUILD_TUPLE', len(node
.bases
))
162 self
.emit('LOAD_CONST', gen
.getCode())
163 self
.emit('MAKE_FUNCTION', 0)
164 self
.emit('CALL_FUNCTION', 0)
165 self
.emit('BUILD_CLASS')
166 self
.storeName(node
.name
)
168 # The rest are standard visitor methods
170 # The next few implement control-flow statements
172 def visitIf(self
, node
):
173 end
= self
.newBlock()
174 numtests
= len(node
.tests
)
175 for i
in range(numtests
):
176 test
, suite
= node
.tests
[i
]
177 self
.set_lineno(test
)
179 ## if i == numtests - 1 and not node.else_:
182 ## nextTest = self.newBlock()
183 nextTest
= self
.newBlock()
184 self
.emit('JUMP_IF_FALSE', nextTest
)
188 self
.emit('JUMP_FORWARD', end
)
189 self
.nextBlock(nextTest
)
192 self
.visit(node
.else_
)
195 def visitWhile(self
, node
):
196 self
.set_lineno(node
)
198 loop
= self
.newBlock()
199 else_
= self
.newBlock()
201 after
= self
.newBlock()
202 self
.emit('SETUP_LOOP', after
)
205 self
.loops
.push(loop
)
207 self
.set_lineno(node
)
208 self
.visit(node
.test
)
209 self
.emit('JUMP_IF_FALSE', else_
or after
)
213 self
.visit(node
.body
)
214 self
.emit('JUMP_ABSOLUTE', loop
)
216 self
.startBlock(else_
) # or just the POPs if not else clause
218 self
.emit('POP_BLOCK')
220 self
.visit(node
.else_
)
222 self
.nextBlock(after
)
224 def visitFor(self
, node
):
225 start
= self
.newBlock()
226 anchor
= self
.newBlock()
227 after
= self
.newBlock()
228 self
.loops
.push(start
)
230 self
.set_lineno(node
)
231 self
.emit('SETUP_LOOP', after
)
232 self
.visit(node
.list)
233 self
.visit(ast
.Const(0))
234 self
.nextBlock(start
)
235 self
.set_lineno(node
)
236 self
.emit('FOR_LOOP', anchor
)
237 self
.visit(node
.assign
)
238 self
.visit(node
.body
)
239 self
.emit('JUMP_ABSOLUTE', start
)
240 self
.nextBlock(anchor
)
241 self
.emit('POP_BLOCK')
243 self
.visit(node
.else_
)
245 self
.nextBlock(after
)
247 def visitBreak(self
, node
):
249 raise SyntaxError, "'break' outside loop (%s, %d)" % \
250 (self
.filename
, node
.lineno
)
251 self
.set_lineno(node
)
252 self
.emit('BREAK_LOOP')
254 def visitContinue(self
, node
):
256 raise SyntaxError, "'continue' outside loop (%s, %d)" % \
257 (self
.filename
, node
.lineno
)
259 self
.set_lineno(node
)
260 self
.emit('JUMP_ABSOLUTE', l
)
263 def visitTest(self
, node
, jump
):
264 end
= self
.newBlock()
265 for child
in node
.nodes
[:-1]:
270 self
.visit(node
.nodes
[-1])
273 def visitAnd(self
, node
):
274 self
.visitTest(node
, 'JUMP_IF_FALSE')
276 def visitOr(self
, node
):
277 self
.visitTest(node
, 'JUMP_IF_TRUE')
279 def visitCompare(self
, node
):
280 self
.visit(node
.expr
)
281 cleanup
= self
.newBlock()
282 for op
, code
in node
.ops
[:-1]:
285 self
.emit('ROT_THREE')
286 self
.emit('COMPARE_OP', op
)
287 self
.emit('JUMP_IF_FALSE', cleanup
)
290 # now do the last comparison
292 op
, code
= node
.ops
[-1]
294 self
.emit('COMPARE_OP', op
)
295 if len(node
.ops
) > 1:
296 end
= self
.newBlock()
297 self
.emit('JUMP_FORWARD', end
)
298 self
.nextBlock(cleanup
)
305 def visitAssert(self
, node
):
306 # XXX would be interesting to implement this via a
307 # transformation of the AST before this stage
308 end
= self
.newBlock()
309 self
.set_lineno(node
)
310 # XXX __debug__ and AssertionError appear to be special cases
311 # -- they are always loaded as globals even if there are local
312 # names. I guess this is a sort of renaming op.
313 self
.emit('LOAD_GLOBAL', '__debug__')
314 self
.emit('JUMP_IF_FALSE', end
)
317 self
.visit(node
.test
)
318 self
.emit('JUMP_IF_TRUE', end
)
320 self
.emit('LOAD_GLOBAL', 'AssertionError')
321 self
.visit(node
.fail
)
322 self
.emit('RAISE_VARARGS', 2)
326 def visitRaise(self
, node
):
327 self
.set_lineno(node
)
330 self
.visit(node
.expr1
)
333 self
.visit(node
.expr2
)
336 self
.visit(node
.expr3
)
338 self
.emit('RAISE_VARARGS', n
)
340 def visitTryExcept(self
, node
):
341 handlers
= self
.newBlock()
342 end
= self
.newBlock()
344 lElse
= self
.newBlock()
347 self
.set_lineno(node
)
348 self
.emit('SETUP_EXCEPT', handlers
)
349 self
.visit(node
.body
)
350 self
.emit('POP_BLOCK')
351 self
.emit('JUMP_FORWARD', lElse
)
352 self
.nextBlock(handlers
)
354 last
= len(node
.handlers
) - 1
355 for i
in range(len(node
.handlers
)):
356 expr
, target
, body
= node
.handlers
[i
]
357 self
.set_lineno(expr
)
361 self
.emit('COMPARE_OP', 'exception match')
362 next
= self
.newBlock()
363 self
.emit('JUMP_IF_FALSE', next
)
373 self
.emit('JUMP_FORWARD', end
)
377 self
.emit('END_FINALLY')
379 self
.nextBlock(lElse
)
380 self
.visit(node
.else_
)
383 def visitTryFinally(self
, node
):
384 final
= self
.newBlock()
385 self
.set_lineno(node
)
386 self
.emit('SETUP_FINALLY', final
)
387 self
.visit(node
.body
)
388 self
.emit('POP_BLOCK')
389 self
.emit('LOAD_CONST', None)
390 self
.nextBlock(final
)
391 self
.visit(node
.final
)
392 self
.emit('END_FINALLY')
396 ## def visitStmt(self, node):
397 ## # nothing to do except walk the children
400 def visitDiscard(self
, node
):
401 self
.visit(node
.expr
)
404 def visitConst(self
, node
):
405 self
.emit('LOAD_CONST', node
.value
)
407 def visitKeyword(self
, node
):
408 self
.emit('LOAD_CONST', node
.name
)
409 self
.visit(node
.expr
)
411 def visitGlobal(self
, node
):
412 # no code to generate
415 def visitName(self
, node
):
416 self
.loadName(node
.name
)
418 def visitPass(self
, node
):
419 self
.set_lineno(node
)
421 def visitImport(self
, node
):
422 self
.set_lineno(node
)
423 for name
in node
.names
:
424 self
.emit('IMPORT_NAME', name
)
427 def visitFrom(self
, node
):
428 self
.set_lineno(node
)
429 self
.emit('IMPORT_NAME', node
.modname
)
430 for name
in node
.names
:
433 self
.emit('IMPORT_FROM', name
)
436 def visitGetattr(self
, node
):
437 self
.visit(node
.expr
)
438 self
.emit('LOAD_ATTR', node
.attrname
)
440 # next five implement assignments
442 def visitAssign(self
, node
):
443 self
.set_lineno(node
)
444 self
.visit(node
.expr
)
445 dups
= len(node
.nodes
) - 1
446 for i
in range(len(node
.nodes
)):
450 if isinstance(elt
, ast
.Node
):
453 def visitAssName(self
, node
):
454 if node
.flags
== 'OP_ASSIGN':
455 self
.storeName(node
.name
)
456 elif node
.flags
== 'OP_DELETE':
457 self
.delName(node
.name
)
459 print "oops", node
.flags
461 def visitAssAttr(self
, node
):
462 self
.visit(node
.expr
)
463 if node
.flags
== 'OP_ASSIGN':
464 self
.emit('STORE_ATTR', node
.attrname
)
465 elif node
.flags
== 'OP_DELETE':
466 self
.emit('DELETE_ATTR', node
.attrname
)
468 print "warning: unexpected flags:", node
.flags
471 def visitAssTuple(self
, node
):
472 if findOp(node
) != 'OP_DELETE':
473 self
.emit('UNPACK_SEQUENCE', len(node
.nodes
))
474 for child
in node
.nodes
:
477 visitAssList
= visitAssTuple
479 def visitExec(self
, node
):
480 self
.visit(node
.expr
)
481 if node
.locals is None:
482 self
.emit('LOAD_CONST', None)
484 self
.visit(node
.locals)
485 if node
.globals is None:
488 self
.visit(node
.globals)
489 self
.emit('EXEC_STMT')
491 def visitCallFunc(self
, node
):
494 self
.set_lineno(node
)
495 self
.visit(node
.node
)
496 for arg
in node
.args
:
498 if isinstance(arg
, ast
.Keyword
):
502 if node
.star_args
is not None:
503 self
.visit(node
.star_args
)
504 if node
.dstar_args
is not None:
505 self
.visit(node
.dstar_args
)
506 have_star
= node
.star_args
is not None
507 have_dstar
= node
.dstar_args
is not None
508 opcode
= callfunc_opcode_info
[have_star
, have_dstar
]
509 self
.emit(opcode
, kw
<< 8 | pos
)
511 def visitPrint(self
, node
):
512 self
.set_lineno(node
)
513 for child
in node
.nodes
:
515 self
.emit('PRINT_ITEM')
517 def visitPrintnl(self
, node
):
518 self
.visitPrint(node
)
519 self
.emit('PRINT_NEWLINE')
521 def visitReturn(self
, node
):
522 self
.set_lineno(node
)
523 self
.visit(node
.value
)
524 self
.emit('RETURN_VALUE')
526 # slice and subscript stuff
528 def visitSlice(self
, node
):
529 self
.visit(node
.expr
)
532 self
.visit(node
.lower
)
535 self
.visit(node
.upper
)
537 if node
.flags
== 'OP_APPLY':
538 self
.emit('SLICE+%d' % slice)
539 elif node
.flags
== 'OP_ASSIGN':
540 self
.emit('STORE_SLICE+%d' % slice)
541 elif node
.flags
== 'OP_DELETE':
542 self
.emit('DELETE_SLICE+%d' % slice)
544 print "weird slice", node
.flags
547 def visitSubscript(self
, node
):
548 self
.visit(node
.expr
)
549 for sub
in node
.subs
:
551 if len(node
.subs
) > 1:
552 self
.emit('BUILD_TUPLE', len(node
.subs
))
553 if node
.flags
== 'OP_APPLY':
554 self
.emit('BINARY_SUBSCR')
555 elif node
.flags
== 'OP_ASSIGN':
556 self
.emit('STORE_SUBSCR')
557 elif node
.flags
== 'OP_DELETE':
558 self
.emit('DELETE_SUBSCR')
562 def binaryOp(self
, node
, op
):
563 self
.visit(node
.left
)
564 self
.visit(node
.right
)
567 def visitAdd(self
, node
):
568 return self
.binaryOp(node
, 'BINARY_ADD')
570 def visitSub(self
, node
):
571 return self
.binaryOp(node
, 'BINARY_SUBTRACT')
573 def visitMul(self
, node
):
574 return self
.binaryOp(node
, 'BINARY_MULTIPLY')
576 def visitDiv(self
, node
):
577 return self
.binaryOp(node
, 'BINARY_DIVIDE')
579 def visitMod(self
, node
):
580 return self
.binaryOp(node
, 'BINARY_MODULO')
582 def visitPower(self
, node
):
583 return self
.binaryOp(node
, 'BINARY_POWER')
585 def visitLeftShift(self
, node
):
586 return self
.binaryOp(node
, 'BINARY_LSHIFT')
588 def visitRightShift(self
, node
):
589 return self
.binaryOp(node
, 'BINARY_RSHIFT')
593 def unaryOp(self
, node
, op
):
594 self
.visit(node
.expr
)
597 def visitInvert(self
, node
):
598 return self
.unaryOp(node
, 'UNARY_INVERT')
600 def visitUnarySub(self
, node
):
601 return self
.unaryOp(node
, 'UNARY_NEGATIVE')
603 def visitUnaryAdd(self
, node
):
604 return self
.unaryOp(node
, 'UNARY_POSITIVE')
606 def visitUnaryInvert(self
, node
):
607 return self
.unaryOp(node
, 'UNARY_INVERT')
609 def visitNot(self
, node
):
610 return self
.unaryOp(node
, 'UNARY_NOT')
612 def visitBackquote(self
, node
):
613 return self
.unaryOp(node
, 'UNARY_CONVERT')
617 def bitOp(self
, nodes
, op
):
619 for node
in nodes
[1:]:
623 def visitBitand(self
, node
):
624 return self
.bitOp(node
.nodes
, 'BINARY_AND')
626 def visitBitor(self
, node
):
627 return self
.bitOp(node
.nodes
, 'BINARY_OR')
629 def visitBitxor(self
, node
):
630 return self
.bitOp(node
.nodes
, 'BINARY_XOR')
632 # object constructors
634 def visitEllipsis(self
, node
):
635 self
.emit('LOAD_CONST', Ellipsis)
637 def visitTuple(self
, node
):
638 for elt
in node
.nodes
:
640 self
.emit('BUILD_TUPLE', len(node
.nodes
))
642 def visitList(self
, node
):
643 for elt
in node
.nodes
:
645 self
.emit('BUILD_LIST', len(node
.nodes
))
647 def visitSliceobj(self
, node
):
648 for child
in node
.nodes
:
650 self
.emit('BUILD_SLICE', len(node
.nodes
))
652 def visitDict(self
, node
):
653 lineno
= getattr(node
, 'lineno', None)
655 set.emit('SET_LINENO', lineno
)
656 self
.emit('BUILD_MAP', 0)
657 for k
, v
in node
.items
:
658 lineno2
= getattr(node
, 'lineno', None)
659 if lineno2
is not None and lineno
!= lineno2
:
660 self
.emit('SET_LINENO', lineno2
)
666 self
.emit('STORE_SUBSCR')
668 class ModuleCodeGenerator(CodeGenerator
):
669 super_init
= CodeGenerator
.__init
__
671 def __init__(self
, filename
):
672 # XXX <module> is ? in compile.c
673 self
.graph
= pyassem
.PyFlowGraph("<module>", filename
)
674 self
.super_init(filename
)
676 class FunctionCodeGenerator(CodeGenerator
):
677 super_init
= CodeGenerator
.__init
__
682 def __init__(self
, func
, filename
, isLambda
=0):
684 klass
= FunctionCodeGenerator
685 name
= "<lambda.%d>" % klass
.lambdaCount
686 klass
.lambdaCount
= klass
.lambdaCount
+ 1
689 args
, hasTupleArg
= generateArgList(func
.argnames
)
690 self
.graph
= pyassem
.PyFlowGraph(name
, filename
, args
,
692 self
.isLambda
= isLambda
693 self
.super_init(filename
)
695 lnf
= walk(func
.code
, LocalNameFinder(args
), 0)
696 self
.locals.push(lnf
.getLocals())
698 self
.graph
.setFlag(CO_VARARGS
)
700 self
.graph
.setFlag(CO_VARKEYWORDS
)
701 self
.set_lineno(func
)
703 self
.generateArgUnpack(func
.argnames
)
706 self
.graph
.startExitBlock()
707 if not self
.isLambda
:
708 self
.emit('LOAD_CONST', None)
709 self
.emit('RETURN_VALUE')
711 def generateArgUnpack(self
, args
):
714 if type(arg
) == types
.TupleType
:
715 self
.emit('LOAD_FAST', '.nested%d' % count
)
717 self
.unpackSequence(arg
)
719 def unpackSequence(self
, tup
):
720 self
.emit('UNPACK_SEQUENCE', len(tup
))
722 if type(elt
) == types
.TupleType
:
723 self
.unpackSequence(elt
)
725 self
.emit('STORE_FAST', elt
)
727 unpackTuple
= unpackSequence
729 class ClassCodeGenerator(CodeGenerator
):
730 super_init
= CodeGenerator
.__init
__
732 def __init__(self
, klass
, filename
):
733 self
.graph
= pyassem
.PyFlowGraph(klass
.name
, filename
,
735 self
.super_init(filename
)
736 lnf
= walk(klass
.code
, LocalNameFinder(), 0)
737 self
.locals.push(lnf
.getLocals())
738 self
.graph
.setFlag(CO_NEWLOCALS
)
741 self
.graph
.startExitBlock()
742 self
.emit('LOAD_LOCALS')
743 self
.emit('RETURN_VALUE')
746 def generateArgList(arglist
):
747 """Generate an arg list marking TupleArgs"""
752 if type(elt
) == types
.StringType
:
754 elif type(elt
) == types
.TupleType
:
755 args
.append(TupleArg(count
, elt
))
757 extra
.extend(misc
.flatten(elt
))
759 raise ValueError, "unexpect argument type:", elt
760 return args
+ extra
, count
762 class LocalNameFinder
:
763 """Find local names in scope"""
764 def __init__(self
, names
=()):
765 self
.names
= misc
.Set()
766 self
.globals = misc
.Set()
771 for elt
in self
.globals.elements():
772 if self
.names
.has_elt(elt
):
773 self
.names
.remove(elt
)
776 def visitDict(self
, node
):
779 def visitGlobal(self
, node
):
780 for name
in node
.names
:
781 self
.globals.add(name
)
783 def visitFunction(self
, node
):
784 self
.names
.add(node
.name
)
786 def visitLambda(self
, node
):
789 def visitImport(self
, node
):
790 for name
in node
.names
:
793 def visitFrom(self
, node
):
794 for name
in node
.names
:
797 def visitClass(self
, node
):
798 self
.names
.add(node
.name
)
800 def visitAssName(self
, node
):
801 self
.names
.add(node
.name
)
804 """Find the op (DELETE, LOAD, STORE) in an AssTuple tree"""
812 def visitAssName(self
, node
):
815 elif self
.op
!= node
.flags
:
816 raise ValueError, "mixed ops in stmt"
818 if __name__
== "__main__":
821 for file in sys
.argv
[1:]: