From 8651beba18eb202d1d7a1debfe660495c6ce1ee3 Mon Sep 17 00:00:00 2001
From: Zach Wegner <zwegner@gmail.com>
Date: Tue, 6 Sep 2011 00:14:51 -0500
Subject: [PATCH] Some cleanup, and add -l option, which adds #line C
 preprocessor directives

---
 prethon.py | 76 ++++++++++++++++++++++++++++----------------------------------
 1 file changed, 34 insertions(+), 42 deletions(-)

diff --git a/prethon.py b/prethon.py
index eb614a1..7ab4af8 100755
--- a/prethon.py
+++ b/prethon.py
@@ -46,7 +46,7 @@ def emit(s):
     pre_state.out.write(str(s))
 
 # Include: Recursively call the preprocessor
-def include(path, var_dict=None, mode=NORMAL):
+def include(path, var_dict=None, mode=NORMAL, output=None):
     global pre_state, depend_files
     depend_files += [path]
     if var_dict:
@@ -54,7 +54,9 @@ def include(path, var_dict=None, mode=NORMAL):
         for key, value in var_dict.items():
             vd[key] = value
         pre_state.variables = vd
-    pre(pre_state.out, pre_state.pre_globals, path, mode=mode)
+    if output is None:
+        output = pre_state.out
+    pre(output, pre_state.pre_globals, path, mode=mode)
 
 def include_py(path, var_dict=None):
     include(path, var_dict, mode=PRE)
@@ -77,19 +79,25 @@ DELIMS = [PRE_START, PRE_END, DEF_START, DEF_END, QUOTE_H_START, QUOTE_H_END,
 
 # Make the reentrant
 class ParserState:
-    def __init__(self, mode):
+    def __init__(self, mode, file, out):
         self.cur_block = []
         self.quote_blocks = []
         self.indent = 0
         self.mode = []
         self.last_mode = -1
         self.last_len = -1
+        self.quote = False
+        self.last_quote = False
+        self.emit = [True]
+        self.path = file
+        self.out = out
+        self.lineno = 1
         self.push(mode)
 
     def push(self, mode):
         # Flush anything from the last mode
         if len(self.mode) >= 1:
-          self.flush(self.mode[-1])
+            self.flush(self.mode[-1])
 
         self.mode.append(mode)
 
@@ -108,27 +116,26 @@ class ParserState:
         self.cur_block.pop()
 
     def flush(self, mode):
+        global output_line_nos
         block = ''.join(self.cur_block.pop())
         self.cur_block.append([])
         s = ''
         if block:
             if mode == NORMAL:
                 s = 'emit(%s)\n' % repr(block)
-                s = self.fix_ws(s)
             elif mode == PRE:
                 s = block
-                s = self.fix_ws(s)
             elif mode == DEF:
                 s = 'emit(%s)\n' % block
-                s = self.fix_ws(s)
             elif mode == QUOTE_H:
-                s = block
-                self.quote_blocks[-1].append(s)
-                s = ''
-            else:
+                self.quote_blocks[-1].append(block)
                 s = ''
 
+        s = self.fix_ws(s)
         self.run(s)
+        if output_line_nos:
+            s = 'emit("\\n#line %s\\n")\n' % self.lineno
+            self.run(s)
 
     def run(self, s):
         # Execute the python code
@@ -188,7 +195,6 @@ def _emit(state, s):
         state.quote_blocks[-1].append(s)
 
 def tokenize(s, delims):
-    tokens = []
     while s:
         idx = None
         t = None
@@ -199,25 +205,18 @@ def tokenize(s, delims):
                 t = d
 
         if t:
-            tokens.append(s[:idx])
-            tokens.append(t)
+            yield s[:idx]
+            yield t
             s = s[idx + len(t):]
         else:
-            tokens.append(s)
+            yield s
             s = ''
 
-    return tokens
-
 def pre(out, pre_globals, file, mode=NORMAL):
     global pre_state
 
     # Set up the state of the parser
-    state = ParserState(mode)
-    state.path = file
-    state.quote = False
-    state.last_quote = False
-    state.out = out
-    state.emit = [True]
+    state = ParserState(mode, file, out)
 
     # Set up globals for the pre-space
     state.pre_globals = pre_globals
@@ -231,9 +230,8 @@ def pre(out, pre_globals, file, mode=NORMAL):
     # Open the file for reading
     with open(file, 'rt') as f:
         for c in f:
-            tokens = tokenize(c, DELIMS)
-
-            for tok in tokens:
+            for tok in tokenize(c, DELIMS):
+                state.lineno += tok.count('\n')
                 # Regular preprocessed sections
                 if tok == PRE_START:
                     state.push(PRE)
@@ -271,11 +269,15 @@ if len(sys.argv) < 3:
 
 depend = None
 depend_files = []
+output_line_nos = False
 
 while True:
     if sys.argv[1] == '-d':
         depend = sys.argv[2]
         sys.argv[1:] = sys.argv[3:]
+    elif sys.argv[1] == '-l':
+        output_line_nos = True
+        sys.argv[1:] = sys.argv[2:]
     else:
         break
 
@@ -297,26 +299,16 @@ p.variables = variables
 
 # Preprocessor globals. This keeps the state of the preprocessed blocks
 pre_globals = {
-        'emit' : emit,
-        'include' : include,
-        'include_py' : include_py,
-        'pre' : p
-        }
+    'emit' : emit,
+    'include' : include,
+    'include_py' : include_py,
+    'pre' : p
+}
 
 # Run the preprocessor
 with open(o, 'wt') as out:
     pre(out, pre_globals, i)
 
 if depend:
-    if os.path.isfile(depend):
-        with open(depend, 'rt') as d_file:
-            lines = d_file.readlines()
-        lines = [l for l in lines if l.strip() and l[:l.find(':')] != o]
-    else:
-        lines = []
-
-    line = '%s: %s' % (o, ' '.join(depend_files))
-    lines += [line]
-
     with open(depend, 'wt') as d_file:
-        d_file.write('\n'.join(lines))
+        d_file.write('%s: %s\n' % (o, ' '.join(depend_files)))
-- 
2.11.4.GIT