prethon.py

   1 ################################################################################
   2 ##
   3 ## Prethon-Python-based preprocessor.
   4 ##
   5 ## Copyright 2011 Zach Wegner
   6 ##
   7 ## This file is part of Prethon.
   8 ##
   9 ## Prethon is free software: you can redistribute it and/or modify
  10 ## it under the terms of the GNU General Public License as published by
  11 ## the Free Software Foundation, either version 3 of the License, or
  12 ## (at your option) any later version.
  13 ##
  14 ## Prethon is distributed in the hope that it will be useful,
  15 ## but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 ## GNU General Public License for more details.
  18 ##
  19 ## You should have received a copy of the GNU General Public License
  20 ## along with Prethon.  If not, see <http://www.gnu.org/licenses/>.
  21 ##
  22 ################################################################################
  23
  24 import copy
  25 import io
  26 import os
  27 import re
  28 import subprocess
  29 import sys
  30
  31 # This state is used by the preprocessor external functions. The preprocessor
  32 # uses its own local state for the parsing, but the preprocessed code needs
  33 # access (through this module) to this state.
  34 pre_state = None
  35
  36 # Mode enum
  37 NORMAL, PRE, DEF, QUOTE_H, QUOTE = range(5)
  38
  39 ################################################################################
  40 ## Preprocessor functions ######################################################
  41 ################################################################################
  42
  43 # Emit function. This is what preprocessor code uses to emit real code.
  44 def emit(s):
  45     global pre_state
  46     pre_state.out.write(str(s))
  47
  48 # Include: Recursively call the preprocessor
  49 def include(path, var_dict=None, mode=NORMAL, output=None):
  50     global pre_state, depend_files
  51     depend_files += [path]
  52     if var_dict:
  53         vd = pre_state.variables.copy()
  54         for key, value in var_dict.items():
  55             vd[key] = value
  56         pre_state.variables = vd
  57     if output is None:
  58         output = pre_state.out
  59     pre(output, pre_state.pre_globals, path, mode=mode)
  60
  61 def include_py(path, var_dict=None):
  62     include(path, var_dict, mode=PRE)
  63
  64 ################################################################################
  65 ## Parser functions ############################################################
  66 ################################################################################
  67
  68 PRE_START = '<@'
  69 PRE_END = '@>'
  70 DEF_START = '<$'
  71 DEF_END = '$>'
  72 QUOTE_H_START = '<#'
  73 QUOTE_H_END = ':'
  74 QUOTE_CONT = '##'
  75 QUOTE_END = '#>'
  76
  77 DELIMS = [PRE_START, PRE_END, DEF_START, DEF_END, QUOTE_H_START, QUOTE_H_END,
  78     QUOTE_CONT, QUOTE_END]
  79
  80 # Make the reentrant
  81 class ParserState:
  82     def __init__(self, mode, file, out):
  83         self.cur_block = []
  84         self.quote_blocks = []
  85         self.indent = 0
  86         self.mode = []
  87         self.last_mode = -1
  88         self.last_len = -1
  89         self.quote = False
  90         self.last_quote = False
  91         self.emit = [True]
  92         self.path = file
  93         self.out = out
  94         self.lineno = 1
  95         self.push(mode)
  96
  97     def push(self, mode):
  98         # Flush anything from the last mode
  99         if len(self.mode) >= 1:
 100             self.flush(self.mode[-1])
 101
 102         self.mode.append(mode)
 103
 104         self.cur_block.append([])
 105         if mode == QUOTE_H:
 106             self.quote_blocks.append([])
 107
 108     def pop(self):
 109         mode = self.mode.pop()
 110         if mode == QUOTE:
 111             s = self.quote_fn(self.quote_blocks.pop())
 112             self.run(s)
 113         else:
 114             self.flush(mode)
 115
 116         self.cur_block.pop()
 117
 118     def flush(self, mode):
 119         global output_line_nos
 120         block = ''.join(self.cur_block.pop())
 121         self.cur_block.append([])
 122         s = ''
 123         if block:
 124             if mode == NORMAL:
 125                 s = 'emit(%s)\n' % repr(block)
 126             elif mode == PRE:
 127                 s = block
 128             elif mode == DEF:
 129                 s = 'emit(%s)\n' % block
 130             elif mode == QUOTE_H:
 131                 self.quote_blocks[-1].append(block)
 132                 s = ''
 133
 134         s = self.fix_ws(s)
 135         self.run(s)
 136         if output_line_nos:
 137             s = 'emit("\\n#line %s\\n")\n' % self.lineno
 138             self.run(s)
 139
 140     def run(self, s):
 141         # Execute the python code
 142         if QUOTE in self.mode:
 143             self.quote_blocks[-1].append(s)
 144         elif s is not '':
 145             try:
 146                 exec(s, self.pre_globals)
 147             except:
 148                 print('Exception in code:\n%s' % s)
 149                 raise
 150
 151     def quote_fn(self, blocks):
 152         header = blocks[0]
 153         body = ''.join(blocks[1:])
 154
 155         header = '%s:\n' % header
 156         header = self.fix_ws(header)
 157
 158         # Set up body
 159         self.indent += 4
 160         body = self.fix_ws(body)
 161         self.indent -= 4
 162
 163         return '\n'.join([header, body])
 164
 165     # Fix the indenting of a block to be at the global scope
 166     def fix_ws(self, block):
 167         lines = block.split('\n')
 168
 169         pre = None
 170         l = 0
 171         for line in lines:
 172             if not line.strip():
 173                 continue
 174             elif pre is None:
 175                 pre = re.match('\\s*', line).group(0)
 176                 l = len(pre)
 177             else:
 178                 for x in range(l):
 179                     if x >= len(line) or line[x] != pre[x]:
 180                         l = x
 181                         break
 182
 183         # Re-indent the lines to match the indent level
 184         lines = [line[l:] if line.strip() else line for line in lines]
 185         lines = [' '*self.indent + line for line in lines]
 186
 187         return '%s\n' % '\n'.join(lines)
 188
 189
 190 # Just add a character to a buffer
 191 def _emit(state, s):
 192     state.cur_block[-1] += [s]
 193     if state.mode[-1] == QUOTE and s:
 194         s = 'emit(%s)\n' % repr(s)
 195         state.quote_blocks[-1].append(s)
 196
 197 def tokenize(s, delims):
 198     while s:
 199         idx = None
 200         t = None
 201         for d in delims:
 202             i = s.find(d)
 203             if i != -1 and (idx is None or i < idx):
 204                 idx = i
 205                 t = d
 206
 207         if t:
 208             yield s[:idx]
 209             yield t
 210             s = s[idx + len(t):]
 211         else:
 212             yield s
 213             s = ''
 214
 215 def pre(out, pre_globals, file, mode=NORMAL):
 216     global pre_state
 217
 218     # Set up the state of the parser
 219     state = ParserState(mode, file, out)
 220
 221     # Set up globals for the pre-space
 222     state.pre_globals = pre_globals
 223
 224     # Set the global state so functions in this module can use it while being
 225     # called from the preprocessed code. We back up the old state since we can
 226     # preprocess recursively (through includes)
 227     old_state = pre_state
 228     pre_state = state
 229
 230     # Open the file for reading
 231     with open(file, 'rt') as f:
 232         for c in f:
 233             for tok in tokenize(c, DELIMS):
 234                 state.lineno += tok.count('\n')
 235                 # Regular preprocessed sections
 236                 if tok == PRE_START:
 237                     state.push(PRE)
 238                 elif tok == PRE_END:
 239                     state.pop()
 240                 # Def
 241                 elif tok == DEF_START:
 242                     state.push(DEF)
 243                 elif tok == DEF_END:
 244                     state.pop()
 245                 # Quote
 246                 elif tok == QUOTE_H_START:
 247                     state.push(QUOTE_H)
 248                 elif tok == QUOTE_H_END and state.mode[-1] == QUOTE_H:
 249                     state.pop()
 250                     state.push(QUOTE)
 251                 elif tok == QUOTE_CONT and state.mode[-1] == QUOTE:
 252                     state.pop()
 253                     state.push(QUOTE_H)
 254                 elif tok == QUOTE_END:
 255                     state.pop()
 256                 else:
 257                     _emit(state, tok)
 258
 259     # Finish up: flush the last block of characters
 260     state.pop()
 261
 262     # Restore the old parser state
 263     pre_state = old_state
 264
 265 # Set up options
 266 if len(sys.argv) < 3:
 267     print('Usage: %s [options] <input> <output> [var=value...]' % sys.argv[0])
 268     sys.exit(1)
 269
 270 depend = None
 271 depend_files = []
 272 output_line_nos = False
 273
 274 while True:
 275     if sys.argv[1] == '-d':
 276         depend = sys.argv[2]
 277         sys.argv[1:] = sys.argv[3:]
 278     elif sys.argv[1] == '-l':
 279         output_line_nos = True
 280         sys.argv[1:] = sys.argv[2:]
 281     else:
 282         break
 283
 284 # Set up input/output files
 285 i = sys.argv[1]
 286 o = sys.argv[2]
 287
 288 # Wrapper class for passing stuff to the program
 289 class PreData: pass
 290
 291 # Loop over all key=value pairs and set these variables.
 292 variables = {}
 293 for opt in sys.argv[3:]:
 294     key, _, value = opt.partition('=')
 295     variables[key] = value
 296
 297 p = PreData()
 298 p.variables = variables
 299
 300 # Preprocessor globals. This keeps the state of the preprocessed blocks
 301 pre_globals = {
 302     'emit' : emit,
 303     'include' : include,
 304     'include_py' : include_py,
 305     'pre' : p
 306 }
 307
 308 # Run the preprocessor
 309 with open(o, 'wt') as out:
 310     pre(out, pre_globals, i)
 311
 312 if depend:
 313     with open(depend, 'wt') as d_file:
 314         d_file.write('%s: %s\n' % (o, ' '.join(depend_files)))