From 21b9eb4837f7887b7280c279ff051418b495ac1f Mon Sep 17 00:00:00 2001 From: "Owen W. Taylor" Date: Fri, 4 Jan 2008 18:01:58 -0500 Subject: [PATCH] Redo capturing of standard output to not use statement rewriting http://www.reinteract.org/trac/ticket/6 - Eduardo de Oliveira Padoan stdout_capture.py: Facility for capturing standard out to a function (thread safely, future-proofing) rewrite.py: Make print-statement rewriting optional statement.py: Use StdoutCapture to implement standard-out capturing. Fix up the test cases which had bit-rotted. --- lib/reinteract/main.py | 3 ++ lib/reinteract/rewrite.py | 90 +++++++++++++++++++---------------- lib/reinteract/shell_buffer.py | 3 ++ lib/reinteract/statement.py | 57 ++++++++++++++++++---- lib/reinteract/stdout_capture.py | 100 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 205 insertions(+), 48 deletions(-) create mode 100644 lib/reinteract/stdout_capture.py diff --git a/lib/reinteract/main.py b/lib/reinteract/main.py index 206a7c5..8a91b15 100644 --- a/lib/reinteract/main.py +++ b/lib/reinteract/main.py @@ -3,6 +3,7 @@ import pango import logging import os +import stdout_capture import sys from notebook import Notebook @@ -12,6 +13,8 @@ from shell_view import ShellView from format_escaped import format_escaped from optparse import OptionParser +stdout_capture.init() + usage = "usage: %prog [options]" op = OptionParser(usage=usage) op.add_option("-u", "--ui", type="choice", choices=("standard", "hildon"), diff --git a/lib/reinteract/rewrite.py b/lib/reinteract/rewrite.py index c879f11..54d7112 100644 --- a/lib/reinteract/rewrite.py +++ b/lib/reinteract/rewrite.py @@ -3,6 +3,16 @@ import token import symbol import sys +class _RewriteState(object): + def __init__(self, output_func_name=None, print_func_name=None): + self.mutated = [] + self.output_func_name = output_func_name + self.print_func_name = print_func_name + + def add_mutated(self, method_spec): + if not method_spec in self.mutated: + self.mutated.append(method_spec) + def _do_match(t, pattern): # Match an AST tree against a pattern. Along with symbol/token names, patterns # can contain strings: @@ -265,7 +275,7 @@ def _create_funccall_expr_stmt(name, args): return _do_create_funccall_expr_stmt(name, trailer) -def _rewrite_tree(t, mutated, actions): +def _rewrite_tree(t, state, actions): # Generic rewriting of an AST, actions is a map of symbol/token type to function # to call to produce a modified version of the the subtree result = t @@ -273,7 +283,7 @@ def _rewrite_tree(t, mutated, actions): subnode = t[i] subtype = subnode[0] if actions.has_key(subtype): - filtered = actions[subtype](subnode, mutated) + filtered = actions[subtype](subnode, state) if filtered != subnode: if result is t: result = list(t) @@ -281,7 +291,7 @@ def _rewrite_tree(t, mutated, actions): return result -def _rewrite_expr_stmt(t, mutated): +def _rewrite_expr_stmt(t, state): # expr_stmt: testlist (augassign (yield_expr|testlist) | # ('=' (yield_expr|testlist))*) @@ -296,10 +306,12 @@ def _rewrite_expr_stmt(t, mutated): if subsubnode[0] == symbol.test: method_spec = _is_test_method_call(subsubnode) if (method_spec != None): - if not method_spec in mutated: - mutated.append(method_spec) - - return _create_funccall_expr_stmt('reinteract_output', filter(lambda x: type(x) != int and x[0] == symbol.test, subnode)) + state.add_mutated(method_spec) + + if state.output_func_name != None: + return _create_funccall_expr_stmt(state.output_func_name, filter(lambda x: type(x) != int and x[0] == symbol.test, subnode)) + else: + return t else: if (t[2][0] == symbol.augassign): # testlist augassign (yield_expr|testlist) @@ -311,8 +323,7 @@ def _rewrite_expr_stmt(t, mutated): variable = _is_test_attribute(subnode[1]) if variable != None: - if not variable in mutated: - mutated.append(variable) + state.add_mutated(variable) else: # testlist ('=' (yield_expr|testlist))+ for i in xrange(1, len(t) - 1): @@ -327,41 +338,40 @@ def _rewrite_expr_stmt(t, mutated): variable = _is_test_attribute(subnode[1]) if variable != None: - if not variable in mutated: - mutated.append(variable) + state.add_mutated(variable) return t -def _rewrite_print_stmt(t, mutated): +def _rewrite_print_stmt(t, state): # print_stmt: 'print' ( [ test (',' test)* [','] ] | # '>>' test [ (',' test)+ [','] ] ) - if t[2][0] == symbol.test: - return _create_funccall_expr_stmt('reinteract_print', filter(lambda x: type(x) != int and x[0] == symbol.test, t)) + if state.print_func_name !=None and t[2][0] == symbol.test: + return _create_funccall_expr_stmt(state.print_func_name, filter(lambda x: type(x) != int and x[0] == symbol.test, t)) else: return t -def _rewrite_small_stmt(t, mutated): +def _rewrite_small_stmt(t, state): # small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | - # import_stmt | global_stmt | exec_stmt | assert_stmt) - return _rewrite_tree(t, mutated, + # import_stmt | global_stmt | exec_stmt | assert_return) + return _rewrite_tree(t, state, { symbol.expr_stmt: _rewrite_expr_stmt, symbol.print_stmt: _rewrite_print_stmt }) # Future special handling: import_stmt # Not valid: flow_stmt, global_stmt -def _rewrite_simple_stmt(t, mutated): +def _rewrite_simple_stmt(t, state): # simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE - return _rewrite_tree(t, mutated, + return _rewrite_tree(t, state, { symbol.small_stmt: _rewrite_small_stmt }) -def _rewrite_suite(t, mutated): +def _rewrite_suite(t, state): # suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT - return _rewrite_tree(t, mutated, + return _rewrite_tree(t, state, { symbol.simple_stmt: _rewrite_simple_stmt, symbol.stmt: _rewrite_stmt }) -def _rewrite_block_stmt(t, mutated): - return _rewrite_tree(t, mutated, +def _rewrite_block_stmt(t, state): + return _rewrite_tree(t, state, { symbol.suite: _rewrite_suite }) _rewrite_compound_stmt_actions = { @@ -376,30 +386,30 @@ if sys.version_info >= (2, 5, 0): _rewrite_compound_stmt_actions[symbol.with_stmt] = _rewrite_block_stmt -def _rewrite_compound_stmt(t, mutated): +def _rewrite_compound_stmt(t, state): # compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef - return _rewrite_tree(t, mutated, _rewrite_compound_stmt_actions) + return _rewrite_tree(t, state, _rewrite_compound_stmt_actions) -def _rewrite_stmt(t, mutated): +def _rewrite_stmt(t, state): # stmt: simple_stmt | compound_stmt - return _rewrite_tree(t, mutated, + return _rewrite_tree(t, state, { symbol.simple_stmt: _rewrite_simple_stmt, symbol.compound_stmt: _rewrite_compound_stmt }) -def _rewrite_file_input(t, mutated): +def _rewrite_file_input(t, state): # file_input: (NEWLINE | stmt)* ENDMARKER - return _rewrite_tree(t, mutated, { symbol.stmt: _rewrite_stmt }) + return _rewrite_tree(t, state, { symbol.stmt: _rewrite_stmt }) -def rewrite_and_compile(code, encoding="utf8"): +def rewrite_and_compile(code, output_func_name=None, print_func_name=None, encoding="utf8"): """ Compiles the supplied text into code, while rewriting the parse tree so: * Print statements without a destination file are transformed into calls to - reinteract_print(*args), + (*args), if print_func_name is not None * Statements which are simply expressions are transformed into calls to - reinteract_output(*args). (More than one argument is passed if the statement - is in the form of a list; for example '1,2'.) + (*args), if output_fnuc_name is not None + (More than one argument is passed if the statement is in the form of a list; for example '1,2'.) At the same time, the code is scanned for possible mutations, and a list is returned. In the list: @@ -411,18 +421,18 @@ def rewrite_and_compile(code, encoding="utf8"): on the variable; this will sometimes be a mutation (e.g., list.append(value)), and sometimes not. """ - mutated = [] + state = _RewriteState(output_func_name=output_func_name, print_func_name=print_func_name) if (isinstance(code, unicode)): code = code.encode("utf8") encoding = "utf8" original = parser.suite(code) - rewritten = _rewrite_file_input(original.totuple(), mutated) + rewritten = _rewrite_file_input(original.totuple(), state) encoded = (symbol.encoding_decl, rewritten, encoding) compiled = parser.sequence2ast(encoded).compile() - return (compiled, mutated) + return (compiled, state.mutated) ##################################################3 @@ -497,7 +507,7 @@ if __name__ == '__main__': # Test that our intercepting of bare expressions to save the output works # def test_output(code, expected): - compiled, _ = rewrite_and_compile(code) + compiled, _ = rewrite_and_compile(code, output_func_name='reinteract_output') test_args = [] def set_test_args(*args): test_args[:] = args @@ -518,7 +528,7 @@ if __name__ == '__main__': # Test that our intercepting of print works # def test_print(code, expected): - compiled, _ = rewrite_and_compile(code) + compiled, _ = rewrite_and_compile(code, print_func_name='reinteract_print') test_args = [] def set_test_args(*args): test_args[:] = args @@ -570,9 +580,9 @@ if __name__ == '__main__': # def test_encoding(code, expected, encoding=None): if encoding != None: - compiled, _ = rewrite_and_compile(code, encoding=encoding) + compiled, _ = rewrite_and_compile(code, encoding=encoding, output_func_name='reinteract_output') else: - compiled, _ = rewrite_and_compile(code) + compiled, _ = rewrite_and_compile(code, output_func_name='reinteract_output') test_args = [] def set_test_args(*args): test_args[:] = args diff --git a/lib/reinteract/shell_buffer.py b/lib/reinteract/shell_buffer.py index d113d29..85ab738 100755 --- a/lib/reinteract/shell_buffer.py +++ b/lib/reinteract/shell_buffer.py @@ -1414,6 +1414,9 @@ class ShellBuffer(gtk.TextBuffer, Worksheet): if __name__ == '__main__': if "-d" in sys.argv: logging.basicConfig(level=logging.DEBUG, format="DEBUG: %(message)s") + + import stdout_capture + stdout_capture.init() S = StatementChunk B = BlankChunk diff --git a/lib/reinteract/statement.py b/lib/reinteract/statement.py index ef723a1..587d2d7 100755 --- a/lib/reinteract/statement.py +++ b/lib/reinteract/statement.py @@ -6,6 +6,7 @@ import sys import rewrite from custom_result import CustomResult from notebook import HelpResult +from stdout_capture import StdoutCapture # A wrapper so we don't have to trap all exceptions when running statement.Execute class ExecutionError(Exception): @@ -27,9 +28,10 @@ class Statement: self.__worksheet = worksheet self.result_scope = None self.results = None + self.stdout_buffer = None # May raise SyntaxError - self.__compiled, self.__mutated = rewrite.rewrite_and_compile(self.__text) + self.__compiled, self.__mutated = rewrite.rewrite_and_compile(self.__text, output_func_name='reinteract_output') self.set_parent(parent) @@ -57,6 +59,23 @@ class Statement: def do_print(self, *args): self.results.append(" ".join(map(str, args))) + def stdout_write(self, str): + if self.stdout_buffer == None: + self.stdout_buffer = str + else: + self.stdout_buffer += str + + pos = 0 + while True: + next = self.stdout_buffer.find("\n", pos) + if next < 0: + break + self.results.append(self.stdout_buffer[pos:next]) + pos = next + 1 + + if pos > 0: + self.stdout_buffer = self.stdout_buffer[pos:] + def execute(self): root_scope = self.__worksheet.global_scope if self.__parent: @@ -66,6 +85,7 @@ class Statement: self.results = [] self.result_scope = scope + self.stdout_buffer = None for mutation in self.__mutated: if isinstance(mutation, tuple): @@ -80,9 +100,13 @@ class Statement: self.results.append(WarningResult("Variable '%s' apparently modified, but can't copy it" % variable)) root_scope['__reinteract_statement'] = self + capture = StdoutCapture(self.stdout_write) + capture.push() try: try: exec self.__compiled in scope, scope + if self.stdout_buffer != None and self.stdout_buffer != '': + self.results.append(self.stdout_buffer) except: self.results = None self.result_scope = None @@ -90,34 +114,51 @@ class Statement: raise ExecutionError(error_type, value, traceback) finally: root_scope['__reinteract_statement'] = None + self.stdout_buffer = None + capture.pop() if __name__=='__main__': + import stdout_capture + from notebook import Notebook + from worksheet import Worksheet + + stdout_capture.init() + + notebook = Notebook() + worksheet = Worksheet(notebook) + def expect(actual,expected): if actual != expected: raise AssertionError("Got: '%s'; Expected: '%s'" % (actual, expected)) def expect_result(text, result): - s = Statement(text) + s = Statement(text, worksheet) s.execute() - expect(s.results[0], result) + if isinstance(result, basestring): + expect(s.results[0], result) + else: + expect(s.results, result) # A bare expression should give the repr of the expression expect_result("'a'", repr('a')) expect_result("1,2", repr((1,2))) - # Print, on the other hand, gives the string form of the expression + # Print, on the other hand, gives the string form of the expression, with + # one result object per output line expect_result("print 'a'", 'a') + expect_result("print 'a', 'b'", ['a b']) + expect_result("print 'a\\nb'", ['a','b']) # Test that we copy a variable before mutating it (when we can detect # the mutation) - s1 = Statement("b = [0]") + s1 = Statement("b = [0]", worksheet) s1.execute() - s2 = Statement("b[0] = 1", parent=s1) + s2 = Statement("b[0] = 1", worksheet, parent=s1) s2.execute() - s3 = Statement("b[0]", parent = s2) + s3 = Statement("b[0]", worksheet, parent = s2) s3.execute() expect(s3.results[0], "1") - s2a = Statement("b[0]", parent=s1) + s2a = Statement("b[0]", worksheet, parent=s1) s2a.execute() expect(s2a.results[0], "0") diff --git a/lib/reinteract/stdout_capture.py b/lib/reinteract/stdout_capture.py new file mode 100644 index 0000000..ce6479e --- /dev/null +++ b/lib/reinteract/stdout_capture.py @@ -0,0 +1,100 @@ +import threading + +import sys + +def init(): + """Initialize the stdout_capture module. This must be called before using the StdoutCapture class""" + sys.stdout = _StdoutStack() + +class _StdoutStack(threading.local): + """The StdoutStack object is used to allow overriding sys.stdout in a per-thread manner""" + + def __init__(self): + self.stack = [] + self.current = sys.stdout + + def write(self, str): + self.current.write(str) + + def push(self, value): + self.stack.append(self.current) + self.current = value + + def pop(self): + self.current = self.stack.pop() + +class StdoutCapture: + """The StdoutCapture object allows temporarily redirecting writes to sys.stdout to call a function + You must call stdout_capture.init() before using this function + + >>> s = "" + >>> def capture_it(str): + ... global s + ... s += str + + >>> c = StdoutCapture(capture_it) + >>> c.push() + >>> try: + ... print "Foo" + ... finally: + ... c.pop() + >>> s + "Foo\n" + + """ + + def __init__(self, write_function): + self.__write_function = write_function + + def push(self): + """Temporarily make the capture object active""" + + if not isinstance(sys.stdout, _StdoutStack): + raise RuntimeError("stdout_capture.init() has not been called, or sys.stdout has been overridden again") + + sys.stdout.push(self) + + def pop(self): + """End the effect of the previous call to pop""" + + if not isinstance(sys.stdout, _StdoutStack): + raise RuntimeError("stdout_capture.init() has not been called, or sys.stdout has been overridden again") + + sys.stdout.pop() + + # Support 'with StdoutCapture(func):' for the future, though reinteract currently limits + # itself to Python-2.4. + + def __enter__(self): + self.push() + + def __exit__(self, *args): + self.pop() + + def write(self, str): + self.__write_function(str) + +if __name__ == "__main__": + init() + + s = "" + def capture_it(str): + global s + s += str + + #with StdoutCapture(capture_it): + # print "Foo" + # + #asssert s == "Foo\n" + + s = "" + + c = StdoutCapture(capture_it) + c.push() + try: + print "Foo" + finally: + c.pop() + + assert s == "Foo\n" + -- 2.11.4.GIT