prethon.py

   1 ################################################################################
   2 ## Prethon-Python-based preprocessor.
   3 ##
   4 ## Copyright 2011 Zach Wegner
   5 ##
   6 ## This file is part of Prethon.
   7 ##
   8 ## Prethon is free software: you can redistribute it and/or modify
   9 ## it under the terms of the GNU General Public License as published by
  10 ## the Free Software Foundation, either version 3 of the License, or
  11 ## (at your option) any later version.
  12 ##
  13 ## Prethon is distributed in the hope that it will be useful,
  14 ## but WITHOUT ANY WARRANTY; without even the implied warranty of
  15 ## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  16 ## GNU General Public License for more details.
  17 ##
  18 ## You should have received a copy of the GNU General Public License
  19 ## along with Prethon.  If not, see <http://www.gnu.org/licenses/>.
  20 ################################################################################
  21
  22 import copy
  23 import io
  24 import os
  25 import re
  26 import subprocess
  27 import sys
  28
  29 # This state is used by the preprocessor external functions. The preprocessor
  30 # uses its own local state for the parsing, but the preprocessed code needs
  31 # access (through this module) to this state.
  32 pre_state = None
  33
  34 # Mode enum
  35 NORMAL, PRE, DEF, QUOTE_H, QUOTE = range(5)
  36
  37 ################################################################################
  38 ## Preprocessor functions ######################################################
  39 ################################################################################
  40
  41 # Emit function. This is what preprocessor code uses to emit real code.
  42 def emit(s):
  43     global pre_state
  44     pre_state.out.write(str(s))
  45
  46 # Include: Recursively call the preprocessor
  47 def include(path, var_dict=None, mode=NORMAL):
  48     global pre_state, depend_files
  49     depend_files += [path]
  50     if var_dict:
  51         vd = pre_state.variables.copy()
  52         for key, value in var_dict.items():
  53             vd[key] = value
  54         pre_state.variables = vd
  55     pre(pre_state.out, pre_state.pre_globals, path, mode=mode)
  56
  57 def include_py(path, var_dict=None):
  58     include(path, var_dict, mode=PRE)
  59
  60 ################################################################################
  61 ## Parser functions ############################################################
  62 ################################################################################
  63
  64 PRE_START = '<@'
  65 PRE_END = '@>'
  66 DEF_START = '<$'
  67 DEF_END = '$>'
  68 QUOTE_H_START = '<#'
  69 QUOTE_H_END = ':'
  70 QUOTE_CONT = '##'
  71 QUOTE_END = '#>'
  72
  73 DELIMS = [PRE_START, PRE_END, DEF_START, DEF_END, QUOTE_H_START, QUOTE_H_END,
  74     QUOTE_CONT, QUOTE_END]
  75
  76 # Make the reentrant
  77 class ParserState:
  78     def __init__(self, mode):
  79         self.cur_block = []
  80         self.quote_blocks = []
  81         self.indent = 0
  82         self.mode = []
  83         self.last_mode = -1
  84         self.last_len = -1
  85         self.push(mode)
  86
  87     def push(self, mode):
  88         # Flush anything from the last mode
  89         if len(self.mode) >= 1:
  90           self.flush(self.mode[-1])
  91
  92         self.mode.append(mode)
  93
  94         self.cur_block.append([])
  95         if mode == QUOTE_H:
  96             self.quote_blocks.append([])
  97
  98     def pop(self):
  99         mode = self.mode.pop()
 100         if mode == QUOTE:
 101             s = self.quote_fn(self.quote_blocks.pop())
 102             self.run(s)
 103         else:
 104             self.flush(mode)
 105
 106         self.cur_block.pop()
 107
 108     def flush(self, mode):
 109         block = ''.join(self.cur_block.pop())
 110         self.cur_block.append([])
 111         s = ''
 112         if block:
 113             if mode == NORMAL:
 114                 s = 'emit(%s)\n' % repr(block)
 115                 s = self.fix_ws(s)
 116             elif mode == PRE:
 117                 s = block
 118                 s = self.fix_ws(s)
 119             elif mode == DEF:
 120                 s = 'emit(%s)\n' % block
 121                 s = self.fix_ws(s)
 122             elif mode == QUOTE_H:
 123                 s = block
 124                 self.quote_blocks[-1].append(s)
 125                 s = ''
 126             else:
 127                 s = ''
 128
 129         self.run(s)
 130
 131     def run(self, s):
 132         # Execute the python code
 133         if QUOTE in self.mode:
 134             self.quote_blocks[-1].append(s)
 135         elif s is not '':
 136             try:
 137                 exec(s, self.pre_globals)
 138             except:
 139                 print('Exception in code:\n%s' % s)
 140                 raise
 141
 142     def quote_fn(self, blocks):
 143         header = blocks[0]
 144         body = ''.join(blocks[1:])
 145
 146         header = '%s:\n' % header
 147         header = self.fix_ws(header)
 148
 149         # Set up body
 150         #body = 'emit(%s)\n' % repr(body)
 151         self.indent += 4
 152         body = self.fix_ws(body)
 153         self.indent -= 4
 154
 155         return '\n'.join([header, body])
 156
 157     # Fix the indenting of a block to be at the global scope
 158     def fix_ws(self, block):
 159         lines = block.split('\n')
 160
 161         pre = None
 162         l = 0
 163         for line in lines:
 164             if not line.strip():
 165                 continue
 166             elif pre is None:
 167                 pre = re.match('\\s*', line).group(0)
 168                 l = len(pre)
 169             else:
 170                 for x in range(l):
 171                     if x >= len(line) or line[x] != pre[x]:
 172                         l = x
 173                         break
 174
 175         # Re-indent the lines to match the indent level
 176         lines = [line[l:] if line.strip() else line for line in lines]
 177         lines = [' '*self.indent + line for line in lines]
 178
 179         return '%s\n' % '\n'.join(lines)
 180
 181
 182 # Just add a character to a buffer
 183 def _emit(state, s):
 184     state.cur_block[-1] += [s]
 185     if QUOTE == state.mode[-1] and s:
 186         s = 'emit(%s)\n' % repr(s)
 187         state.quote_blocks[-1].append(s)
 188
 189 def tokenize(s, delims):
 190     tokens = []
 191     while s:
 192         idx = None
 193         t = None
 194         for d in delims:
 195             i = s.find(d)
 196             if i != -1 and (idx is None or i < idx):
 197                 idx = i
 198                 t = d
 199
 200         if t:
 201             tokens.append(s[:idx])
 202             tokens.append(t)
 203             s = s[idx + len(t):]
 204         else:
 205             tokens.append(s)
 206             s = ''
 207
 208     return tokens
 209
 210 def pre(out, pre_globals, file, mode=NORMAL):
 211     global pre_state
 212
 213     # Set up the state of the parser
 214     state = ParserState(mode)
 215     state.path = file
 216     state.quote = False
 217     state.last_quote = False
 218     state.out = out
 219     state.emit = [True]
 220
 221     # Set up globals for the pre-space
 222     state.pre_globals = pre_globals
 223
 224     # Set the global state so functions in this module can use it while being
 225     # called from the preprocessed code. We back up the old state since we can
 226     # preprocess recursively (through includes)
 227     old_state = pre_state
 228     pre_state = state
 229
 230     # Open the file for reading
 231     with open(file, 'rt') as f:
 232         for c in f:
 233             #tokens = re.findall(pattern, c, re.DOTALL) # DOTALL means keep newlines
 234             tokens = tokenize(c, DELIMS)
 235
 236             for tok in tokens:
 237                 # Regular preprocessed sections
 238                 if tok == PRE_START:
 239                     state.push(PRE)
 240                 elif tok == PRE_END:
 241                     state.pop()
 242                 # Def
 243                 elif tok == DEF_START:
 244                     state.push(DEF)
 245                 elif tok == DEF_END:
 246                     state.pop()
 247                 # Quote
 248                 elif tok == QUOTE_H_START:
 249                     state.push(QUOTE_H)
 250                 elif tok == QUOTE_H_END and state.mode[-1] == QUOTE_H:
 251                     state.pop()
 252                     state.push(QUOTE)
 253                 elif tok == QUOTE_CONT and state.mode[-1] == QUOTE:
 254                     state.pop()
 255                     state.push(QUOTE_H)
 256                 elif tok == QUOTE_END:
 257                     state.pop()
 258                 else:
 259                     _emit(state, tok)
 260
 261     # Finish up: flush the last block of characters
 262     state.pop()
 263
 264     # Restore the old parser state
 265     pre_state = old_state
 266
 267 # Set up options
 268 if len(sys.argv) < 3:
 269     print('Usage: %s [options] <input> <output> [var=value...]' % sys.argv[0])
 270     sys.exit(1)
 271
 272 depend = None
 273 depend_files = []
 274
 275 while True:
 276     if sys.argv[1] == '-d':
 277         depend = sys.argv[2]
 278         sys.argv[1:] = sys.argv[3:]
 279     else:
 280         break
 281
 282 # Set up input/output files
 283 i = sys.argv[1]
 284 o = sys.argv[2]
 285
 286 # Wrapper class for passing stuff to the program
 287 class PreData: pass
 288
 289 # Loop over all key=value pairs and set these variables.
 290 variables = {}
 291 for opt in sys.argv[3:]:
 292     key, _, value = opt.partition('=')
 293     variables[key] = value
 294
 295 p = PreData()
 296 p.variables = variables
 297
 298 # Preprocessor globals. This keeps the state of the preprocessed blocks
 299 pre_globals = {
 300         'emit' : emit,
 301         'include' : include,
 302         'include_py' : include_py,
 303         'pre' : p
 304         }
 305
 306 # Run the preprocessor
 307 with open(o, 'wt') as out:
 308     pre(out, pre_globals, i)
 309
 310 if depend:
 311     if os.path.isfile(depend):
 312         with open(depend, 'rt') as d_file:
 313             lines = d_file.readlines()
 314         lines = [l for l in lines if l.strip() and l[:l.find(':')] != o]
 315     else:
 316         lines = []
 317
 318     line = '%s: %s' % (o, ' '.join(depend_files))
 319     lines += [line]
 320
 321     with open(depend, 'wt') as d_file:
 322         d_file.write('\n'.join(lines))