Tools/compiler/astgen.py

   1 """Generate ast module from specification
   2
   3 This script generates the ast module from a simple specification,
   4 which makes it easy to accomodate changes in the grammar.  This
   5 approach would be quite reasonable if the grammar changed often.
   6 Instead, it is rather complex to generate the appropriate code.  And
   7 the Node interface has changed more often than the grammar.
   8 """
   9
  10 import fileinput
  11 import getopt
  12 import re
  13 import sys
  14 from StringIO import StringIO
  15
  16 SPEC = "ast.txt"
  17 COMMA = ", "
  18
  19 def load_boilerplate(file):
  20     f = open(file)
  21     buf = f.read()
  22     f.close()
  23     i = buf.find('### ''PROLOGUE')
  24     j = buf.find('### ''EPILOGUE')
  25     pro = buf[i+12:j].strip()
  26     epi = buf[j+12:].strip()
  27     return pro, epi
  28
  29 def strip_default(arg):
  30     """Return the argname from an 'arg = default' string"""
  31     i = arg.find('=')
  32     if i == -1:
  33         return arg
  34     t = arg[:i].strip()
  35     return t
  36
  37 P_NODE = 1
  38 P_OTHER = 2
  39 P_NESTED = 3
  40 P_NONE = 4
  41
  42 class NodeInfo:
  43     """Each instance describes a specific AST node"""
  44     def __init__(self, name, args):
  45         self.name = name
  46         self.args = args.strip()
  47         self.argnames = self.get_argnames()
  48         self.argprops = self.get_argprops()
  49         self.nargs = len(self.argnames)
  50         self.init = []
  51
  52     def get_argnames(self):
  53         if '(' in self.args:
  54             i = self.args.find('(')
  55             j = self.args.rfind(')')
  56             args = self.args[i+1:j]
  57         else:
  58             args = self.args
  59         return [strip_default(arg.strip())
  60                 for arg in args.split(',') if arg]
  61
  62     def get_argprops(self):
  63         """Each argument can have a property like '*' or '!'
  64
  65         XXX This method modifies the argnames in place!
  66         """
  67         d = {}
  68         hardest_arg = P_NODE
  69         for i in range(len(self.argnames)):
  70             arg = self.argnames[i]
  71             if arg.endswith('*'):
  72                 arg = self.argnames[i] = arg[:-1]
  73                 d[arg] = P_OTHER
  74                 hardest_arg = max(hardest_arg, P_OTHER)
  75             elif arg.endswith('!'):
  76                 arg = self.argnames[i] = arg[:-1]
  77                 d[arg] = P_NESTED
  78                 hardest_arg = max(hardest_arg, P_NESTED)
  79             elif arg.endswith('&'):
  80                 arg = self.argnames[i] = arg[:-1]
  81                 d[arg] = P_NONE
  82                 hardest_arg = max(hardest_arg, P_NONE)
  83             else:
  84                 d[arg] = P_NODE
  85         self.hardest_arg = hardest_arg
  86
  87         if hardest_arg > P_NODE:
  88             self.args = self.args.replace('*', '')
  89             self.args = self.args.replace('!', '')
  90             self.args = self.args.replace('&', '')
  91
  92         return d
  93
  94     def gen_source(self):
  95         buf = StringIO()
  96         print >> buf, "class %s(Node):" % self.name
  97         print >> buf, '    nodes["%s"] = "%s"' % (self.name.lower(), self.name)
  98         self._gen_init(buf)
  99         print >> buf
 100         self._gen_getChildren(buf)
 101         print >> buf
 102         self._gen_getChildNodes(buf)
 103         print >> buf
 104         self._gen_repr(buf)
 105         buf.seek(0, 0)
 106         return buf.read()
 107
 108     def _gen_init(self, buf):
 109         print >> buf, "    def __init__(self, %s):" % self.args
 110         if self.argnames:
 111             for name in self.argnames:
 112                 print >> buf, "        self.%s = %s" % (name, name)
 113         else:
 114             print >> buf, "        pass"
 115         if self.init:
 116             print >> buf, "".join(["    " + line for line in self.init])
 117
 118     def _gen_getChildren(self, buf):
 119         print >> buf, "    def getChildren(self):"
 120         if len(self.argnames) == 0:
 121             print >> buf, "        return ()"
 122         else:
 123             if self.hardest_arg < P_NESTED:
 124                 clist = COMMA.join(["self.%s" % c
 125                                     for c in self.argnames])
 126                 if self.nargs == 1:
 127                     print >> buf, "        return %s," % clist
 128                 else:
 129                     print >> buf, "        return %s" % clist
 130             else:
 131                 print >> buf, "        children = []"
 132                 template = "        children.%s(%sself.%s%s)"
 133                 for name in self.argnames:
 134                     if self.argprops[name] == P_NESTED:
 135                         print >> buf, template % ("extend", "flatten(",
 136                                                   name, ")")
 137                     else:
 138                         print >> buf, template % ("append", "", name, "")
 139                 print >> buf, "        return tuple(children)"
 140
 141     def _gen_getChildNodes(self, buf):
 142         print >> buf, "    def getChildNodes(self):"
 143         if len(self.argnames) == 0:
 144             print >> buf, "        return ()"
 145         else:
 146             if self.hardest_arg < P_NESTED:
 147                 clist = ["self.%s" % c
 148                          for c in self.argnames
 149                          if self.argprops[c] == P_NODE]
 150                 if len(clist) == 0:
 151                     print >> buf, "        return ()"
 152                 elif len(clist) == 1:
 153                     print >> buf, "        return %s," % clist[0]
 154                 else:
 155                     print >> buf, "        return %s" % COMMA.join(clist)
 156             else:
 157                 print >> buf, "        nodes = []"
 158                 template = "        nodes.%s(%sself.%s%s)"
 159                 for name in self.argnames:
 160                     if self.argprops[name] == P_NONE:
 161                         tmp = ("        if self.%s is not None:"
 162                                "            nodes.append(self.%s)")
 163                         print >> buf, tmp % (name, name)
 164                     elif self.argprops[name] == P_NESTED:
 165                         print >> buf, template % ("extend", "flatten_nodes(",
 166                                                   name, ")")
 167                     elif self.argprops[name] == P_NODE:
 168                         print >> buf, template % ("append", "", name, "")
 169                 print >> buf, "        return tuple(nodes)"
 170
 171     def _gen_repr(self, buf):
 172         print >> buf, "    def __repr__(self):"
 173         if self.argnames:
 174             fmt = COMMA.join(["%s"] * self.nargs)
 175             if '(' in self.args:
 176                 fmt = '(%s)' % fmt
 177             vals = ["repr(self.%s)" % name for name in self.argnames]
 178             vals = COMMA.join(vals)
 179             if self.nargs == 1:
 180                 vals = vals + ","
 181             print >> buf, '        return "%s(%s)" %% (%s)' % \
 182                   (self.name, fmt, vals)
 183         else:
 184             print >> buf, '        return "%s()"' % self.name
 185
 186 rx_init = re.compile('init\((.*)\):')
 187
 188 def parse_spec(file):
 189     classes = {}
 190     cur = None
 191     for line in fileinput.input(file):
 192         if line.strip().startswith('#'):
 193             continue
 194         mo = rx_init.search(line)
 195         if mo is None:
 196             if cur is None:
 197                 # a normal entry
 198                 try:
 199                     name, args = line.split(':')
 200                 except ValueError:
 201                     continue
 202                 classes[name] = NodeInfo(name, args)
 203                 cur = None
 204             else:
 205                 # some code for the __init__ method
 206                 cur.init.append(line)
 207         else:
 208             # some extra code for a Node's __init__ method
 209             name = mo.group(1)
 210             cur = classes[name]
 211     return classes.values()
 212
 213 def main():
 214     prologue, epilogue = load_boilerplate(sys.argv[-1])
 215     print prologue
 216     print
 217     classes = parse_spec(SPEC)
 218     for info in classes:
 219         print info.gen_source()
 220     print epilogue
 221
 222 if __name__ == "__main__":
 223     main()
 224     sys.exit(0)
 225
 226 ### PROLOGUE
 227 """Python abstract syntax node definitions
 228
 229 This file is automatically generated.
 230 """
 231 from types import TupleType, ListType
 232 from consts import CO_VARARGS, CO_VARKEYWORDS
 233
 234 def flatten(list):
 235     l = []
 236     for elt in list:
 237         t = type(elt)
 238         if t is TupleType or t is ListType:
 239             for elt2 in flatten(elt):
 240                 l.append(elt2)
 241         else:
 242             l.append(elt)
 243     return l
 244
 245 def flatten_nodes(list):
 246     return [n for n in flatten(list) if isinstance(n, Node)]
 247
 248 def asList(nodes):
 249     l = []
 250     for item in nodes:
 251         if hasattr(item, "asList"):
 252             l.append(item.asList())
 253         else:
 254             t = type(item)
 255             if t is TupleType or t is ListType:
 256                 l.append(tuple(asList(item)))
 257             else:
 258                 l.append(item)
 259     return l
 260
 261 nodes = {}
 262
 263 class Node: # an abstract base class
 264     lineno = None # provide a lineno for nodes that don't have one
 265     def getType(self):
 266         pass # implemented by subclass
 267     def getChildren(self):
 268         pass # implemented by subclasses
 269     def asList(self):
 270         return tuple(asList(self.getChildren()))
 271     def getChildNodes(self):
 272         pass # implemented by subclasses
 273
 274 class EmptyNode(Node):
 275     pass
 276
 277 ### EPILOGUE
 278 klasses = globals()
 279 for k in nodes.keys():
 280     nodes[k] = klasses[nodes[k]]