prethon.py

   1 ################################################################################
   2 ##
   3 ## Prethon-Python-based preprocessor.
   4 ##
   5 ## Copyright 2011 Zach Wegner
   6 ##
   7 ## This file is part of Prethon.
   8 ##
   9 ## Prethon is free software: you can redistribute it and/or modify
  10 ## it under the terms of the GNU General Public License as published by
  11 ## the Free Software Foundation, either version 3 of the License, or
  12 ## (at your option) any later version.
  13 ##
  14 ## Prethon is distributed in the hope that it will be useful,
  15 ## but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17 ## GNU General Public License for more details.
  18 ##
  19 ## You should have received a copy of the GNU General Public License
  20 ## along with Prethon.  If not, see <http://www.gnu.org/licenses/>.
  21 ##
  22 ################################################################################
  23
  24 import copy
  25 import io
  26 import os
  27 import re
  28 import subprocess
  29 import sys
  30
  31 # This state is used by the preprocessor external functions. The preprocessor
  32 # uses its own local state for the parsing, but the preprocessed code needs
  33 # access (through this module) to this state.
  34 pre_state = None
  35
  36 # Mode enum
  37 NORMAL, PRE, DEF, QUOTE_H, QUOTE = range(5)
  38
  39 ################################################################################
  40 ## Preprocessor functions ######################################################
  41 ################################################################################
  42
  43 # Emit function. This is what preprocessor code uses to emit real code.
  44 def emit(s):
  45     global pre_state
  46     pre_state.out.write(str(s))
  47
  48 # Include: Recursively call the preprocessor
  49 def include(path, var_dict=None, mode=NORMAL):
  50     global pre_state, depend_files
  51     depend_files += [path]
  52     if var_dict:
  53         vd = pre_state.variables.copy()
  54         for key, value in var_dict.items():
  55             vd[key] = value
  56         pre_state.variables = vd
  57     pre(pre_state.out, pre_state.pre_globals, path, mode=mode)
  58
  59 def include_py(path, var_dict=None):
  60     include(path, var_dict, mode=PRE)
  61
  62 ################################################################################
  63 ## Parser functions ############################################################
  64 ################################################################################
  65
  66 PRE_START = '<@'
  67 PRE_END = '@>'
  68 DEF_START = '<$'
  69 DEF_END = '$>'
  70 QUOTE_H_START = '<#'
  71 QUOTE_H_END = ':'
  72 QUOTE_CONT = '##'
  73 QUOTE_END = '#>'
  74
  75 DELIMS = [PRE_START, PRE_END, DEF_START, DEF_END, QUOTE_H_START, QUOTE_H_END,
  76     QUOTE_CONT, QUOTE_END]
  77
  78 # Make the reentrant
  79 class ParserState:
  80     def __init__(self, mode):
  81         self.cur_block = []
  82         self.quote_blocks = []
  83         self.indent = 0
  84         self.mode = []
  85         self.last_mode = -1
  86         self.last_len = -1
  87         self.push(mode)
  88
  89     def push(self, mode):
  90         # Flush anything from the last mode
  91         if len(self.mode) >= 1:
  92           self.flush(self.mode[-1])
  93
  94         self.mode.append(mode)
  95
  96         self.cur_block.append([])
  97         if mode == QUOTE_H:
  98             self.quote_blocks.append([])
  99
 100     def pop(self):
 101         mode = self.mode.pop()
 102         if mode == QUOTE:
 103             s = self.quote_fn(self.quote_blocks.pop())
 104             self.run(s)
 105         else:
 106             self.flush(mode)
 107
 108         self.cur_block.pop()
 109
 110     def flush(self, mode):
 111         block = ''.join(self.cur_block.pop())
 112         self.cur_block.append([])
 113         s = ''
 114         if block:
 115             if mode == NORMAL:
 116                 s = 'emit(%s)\n' % repr(block)
 117                 s = self.fix_ws(s)
 118             elif mode == PRE:
 119                 s = block
 120                 s = self.fix_ws(s)
 121             elif mode == DEF:
 122                 s = 'emit(%s)\n' % block
 123                 s = self.fix_ws(s)
 124             elif mode == QUOTE_H:
 125                 s = block
 126                 self.quote_blocks[-1].append(s)
 127                 s = ''
 128             else:
 129                 s = ''
 130
 131         self.run(s)
 132
 133     def run(self, s):
 134         # Execute the python code
 135         if QUOTE in self.mode:
 136             self.quote_blocks[-1].append(s)
 137         elif s is not '':
 138             try:
 139                 exec(s, self.pre_globals)
 140             except:
 141                 print('Exception in code:\n%s' % s)
 142                 raise
 143
 144     def quote_fn(self, blocks):
 145         header = blocks[0]
 146         body = ''.join(blocks[1:])
 147
 148         header = '%s:\n' % header
 149         header = self.fix_ws(header)
 150
 151         # Set up body
 152         self.indent += 4
 153         body = self.fix_ws(body)
 154         self.indent -= 4
 155
 156         return '\n'.join([header, body])
 157
 158     # Fix the indenting of a block to be at the global scope
 159     def fix_ws(self, block):
 160         lines = block.split('\n')
 161
 162         pre = None
 163         l = 0
 164         for line in lines:
 165             if not line.strip():
 166                 continue
 167             elif pre is None:
 168                 pre = re.match('\\s*', line).group(0)
 169                 l = len(pre)
 170             else:
 171                 for x in range(l):
 172                     if x >= len(line) or line[x] != pre[x]:
 173                         l = x
 174                         break
 175
 176         # Re-indent the lines to match the indent level
 177         lines = [line[l:] if line.strip() else line for line in lines]
 178         lines = [' '*self.indent + line for line in lines]
 179
 180         return '%s\n' % '\n'.join(lines)
 181
 182
 183 # Just add a character to a buffer
 184 def _emit(state, s):
 185     state.cur_block[-1] += [s]
 186     if state.mode[-1] == QUOTE and s:
 187         s = 'emit(%s)\n' % repr(s)
 188         state.quote_blocks[-1].append(s)
 189
 190 def tokenize(s, delims):
 191     tokens = []
 192     while s:
 193         idx = None
 194         t = None
 195         for d in delims:
 196             i = s.find(d)
 197             if i != -1 and (idx is None or i < idx):
 198                 idx = i
 199                 t = d
 200
 201         if t:
 202             tokens.append(s[:idx])
 203             tokens.append(t)
 204             s = s[idx + len(t):]
 205         else:
 206             tokens.append(s)
 207             s = ''
 208
 209     return tokens
 210
 211 def pre(out, pre_globals, file, mode=NORMAL):
 212     global pre_state
 213
 214     # Set up the state of the parser
 215     state = ParserState(mode)
 216     state.path = file
 217     state.quote = False
 218     state.last_quote = False
 219     state.out = out
 220     state.emit = [True]
 221
 222     # Set up globals for the pre-space
 223     state.pre_globals = pre_globals
 224
 225     # Set the global state so functions in this module can use it while being
 226     # called from the preprocessed code. We back up the old state since we can
 227     # preprocess recursively (through includes)
 228     old_state = pre_state
 229     pre_state = state
 230
 231     # Open the file for reading
 232     with open(file, 'rt') as f:
 233         for c in f:
 234             tokens = tokenize(c, DELIMS)
 235
 236             for tok in tokens:
 237                 # Regular preprocessed sections
 238                 if tok == PRE_START:
 239                     state.push(PRE)
 240                 elif tok == PRE_END:
 241                     state.pop()
 242                 # Def
 243                 elif tok == DEF_START:
 244                     state.push(DEF)
 245                 elif tok == DEF_END:
 246                     state.pop()
 247                 # Quote
 248                 elif tok == QUOTE_H_START:
 249                     state.push(QUOTE_H)
 250                 elif tok == QUOTE_H_END and state.mode[-1] == QUOTE_H:
 251                     state.pop()
 252                     state.push(QUOTE)
 253                 elif tok == QUOTE_CONT and state.mode[-1] == QUOTE:
 254                     state.pop()
 255                     state.push(QUOTE_H)
 256                 elif tok == QUOTE_END:
 257                     state.pop()
 258                 else:
 259                     _emit(state, tok)
 260
 261     # Finish up: flush the last block of characters
 262     state.pop()
 263
 264     # Restore the old parser state
 265     pre_state = old_state
 266
 267 # Set up options
 268 if len(sys.argv) < 3:
 269     print('Usage: %s [options] <input> <output> [var=value...]' % sys.argv[0])
 270     sys.exit(1)
 271
 272 depend = None
 273 depend_files = []
 274
 275 while True:
 276     if sys.argv[1] == '-d':
 277         depend = sys.argv[2]
 278         sys.argv[1:] = sys.argv[3:]
 279     else:
 280         break
 281
 282 # Set up input/output files
 283 i = sys.argv[1]
 284 o = sys.argv[2]
 285
 286 # Wrapper class for passing stuff to the program
 287 class PreData: pass
 288
 289 # Loop over all key=value pairs and set these variables.
 290 variables = {}
 291 for opt in sys.argv[3:]:
 292     key, _, value = opt.partition('=')
 293     variables[key] = value
 294
 295 p = PreData()
 296 p.variables = variables
 297
 298 # Preprocessor globals. This keeps the state of the preprocessed blocks
 299 pre_globals = {
 300         'emit' : emit,
 301         'include' : include,
 302         'include_py' : include_py,
 303         'pre' : p
 304         }
 305
 306 # Run the preprocessor
 307 with open(o, 'wt') as out:
 308     pre(out, pre_globals, i)
 309
 310 if depend:
 311     if os.path.isfile(depend):
 312         with open(depend, 'rt') as d_file:
 313             lines = d_file.readlines()
 314         lines = [l for l in lines if l.strip() and l[:l.find(':')] != o]
 315     else:
 316         lines = []
 317
 318     line = '%s: %s' % (o, ' '.join(depend_files))
 319     lines += [line]
 320
 321     with open(depend, 'wt') as d_file:
 322         d_file.write('\n'.join(lines))