Tools/compiler/astgen.py

   1 """Generate ast module from specification
   2
   3 This script generates the ast module from a simple specification,
   4 which makes it easy to accomodate changes in the grammar.  This
   5 approach would be quite reasonable if the grammar changed often.
   6 Instead, it is rather complex to generate the appropriate code.  And
   7 the Node interface has changed more often than the grammar.
   8 """
   9
  10 import fileinput
  11 import re
  12 import sys
  13 from StringIO import StringIO
  14
  15 SPEC = "ast.txt"
  16 COMMA = ", "
  17
  18 def load_boilerplate(file):
  19     f = open(file)
  20     buf = f.read()
  21     f.close()
  22     i = buf.find('### ''PROLOGUE')
  23     j = buf.find('### ''EPILOGUE')
  24     pro = buf[i+12:j].strip()
  25     epi = buf[j+12:].strip()
  26     return pro, epi
  27
  28 def strip_default(arg):
  29     """Return the argname from an 'arg = default' string"""
  30     i = arg.find('=')
  31     if i == -1:
  32         return arg
  33     t = arg[:i].strip()
  34     return t
  35
  36 P_NODE = 1
  37 P_OTHER = 2
  38 P_NESTED = 3
  39 P_NONE = 4
  40
  41 class NodeInfo:
  42     """Each instance describes a specific AST node"""
  43     def __init__(self, name, args):
  44         self.name = name
  45         self.args = args.strip()
  46         self.argnames = self.get_argnames()
  47         self.argprops = self.get_argprops()
  48         self.nargs = len(self.argnames)
  49         self.init = []
  50
  51     def get_argnames(self):
  52         if '(' in self.args:
  53             i = self.args.find('(')
  54             j = self.args.rfind(')')
  55             args = self.args[i+1:j]
  56         else:
  57             args = self.args
  58         return [strip_default(arg.strip())
  59                 for arg in args.split(',') if arg]
  60
  61     def get_argprops(self):
  62         """Each argument can have a property like '*' or '!'
  63
  64         XXX This method modifies the argnames in place!
  65         """
  66         d = {}
  67         hardest_arg = P_NODE
  68         for i in range(len(self.argnames)):
  69             arg = self.argnames[i]
  70             if arg.endswith('*'):
  71                 arg = self.argnames[i] = arg[:-1]
  72                 d[arg] = P_OTHER
  73                 hardest_arg = max(hardest_arg, P_OTHER)
  74             elif arg.endswith('!'):
  75                 arg = self.argnames[i] = arg[:-1]
  76                 d[arg] = P_NESTED
  77                 hardest_arg = max(hardest_arg, P_NESTED)
  78             elif arg.endswith('&'):
  79                 arg = self.argnames[i] = arg[:-1]
  80                 d[arg] = P_NONE
  81                 hardest_arg = max(hardest_arg, P_NONE)
  82             else:
  83                 d[arg] = P_NODE
  84         self.hardest_arg = hardest_arg
  85
  86         if hardest_arg > P_NODE:
  87             self.args = self.args.replace('*', '')
  88             self.args = self.args.replace('!', '')
  89             self.args = self.args.replace('&', '')
  90
  91         return d
  92
  93     def gen_source(self):
  94         buf = StringIO()
  95         print >> buf, "class %s(Node):" % self.name
  96         self._gen_init(buf)
  97         print >> buf
  98         self._gen_getChildren(buf)
  99         print >> buf
 100         self._gen_getChildNodes(buf)
 101         print >> buf
 102         self._gen_repr(buf)
 103         buf.seek(0, 0)
 104         return buf.read()
 105
 106     def _gen_init(self, buf):
 107         if self.args:
 108             argtuple = '(' in self.args
 109             args = self.args if not argtuple else ''.join(self.argnames)
 110             print >> buf, "    def __init__(self, %s, lineno=None):" % args
 111         else:
 112             print >> buf, "    def __init__(self, lineno=None):"
 113         if self.argnames:
 114             if argtuple:
 115                 for idx, name in enumerate(self.argnames):
 116                     print >> buf, "        self.%s = %s[%s]" % (name, args, idx)
 117             else:
 118                 for name in self.argnames:
 119                     print >> buf, "        self.%s = %s" % (name, name)
 120         print >> buf, "        self.lineno = lineno"
 121         # Copy the lines in self.init, indented four spaces.  The rstrip()
 122         # business is to get rid of the four spaces if line happens to be
 123         # empty, so that reindent.py is happy with the output.
 124         for line in self.init:
 125             print >> buf, ("    " + line).rstrip()
 126
 127     def _gen_getChildren(self, buf):
 128         print >> buf, "    def getChildren(self):"
 129         if len(self.argnames) == 0:
 130             print >> buf, "        return ()"
 131         else:
 132             if self.hardest_arg < P_NESTED:
 133                 clist = COMMA.join(["self.%s" % c
 134                                     for c in self.argnames])
 135                 if self.nargs == 1:
 136                     print >> buf, "        return %s," % clist
 137                 else:
 138                     print >> buf, "        return %s" % clist
 139             else:
 140                 if len(self.argnames) == 1:
 141                     print >> buf, "        return tuple(flatten(self.%s))" % self.argnames[0]
 142                 else:
 143                     print >> buf, "        children = []"
 144                     template = "        children.%s(%sself.%s%s)"
 145                     for name in self.argnames:
 146                         if self.argprops[name] == P_NESTED:
 147                             print >> buf, template % ("extend", "flatten(",
 148                                                       name, ")")
 149                         else:
 150                             print >> buf, template % ("append", "", name, "")
 151                     print >> buf, "        return tuple(children)"
 152
 153     def _gen_getChildNodes(self, buf):
 154         print >> buf, "    def getChildNodes(self):"
 155         if len(self.argnames) == 0:
 156             print >> buf, "        return ()"
 157         else:
 158             if self.hardest_arg < P_NESTED:
 159                 clist = ["self.%s" % c
 160                          for c in self.argnames
 161                          if self.argprops[c] == P_NODE]
 162                 if len(clist) == 0:
 163                     print >> buf, "        return ()"
 164                 elif len(clist) == 1:
 165                     print >> buf, "        return %s," % clist[0]
 166                 else:
 167                     print >> buf, "        return %s" % COMMA.join(clist)
 168             else:
 169                 print >> buf, "        nodelist = []"
 170                 template = "        nodelist.%s(%sself.%s%s)"
 171                 for name in self.argnames:
 172                     if self.argprops[name] == P_NONE:
 173                         tmp = ("        if self.%s is not None:\n"
 174                                "            nodelist.append(self.%s)")
 175                         print >> buf, tmp % (name, name)
 176                     elif self.argprops[name] == P_NESTED:
 177                         print >> buf, template % ("extend", "flatten_nodes(",
 178                                                   name, ")")
 179                     elif self.argprops[name] == P_NODE:
 180                         print >> buf, template % ("append", "", name, "")
 181                 print >> buf, "        return tuple(nodelist)"
 182
 183     def _gen_repr(self, buf):
 184         print >> buf, "    def __repr__(self):"
 185         if self.argnames:
 186             fmt = COMMA.join(["%s"] * self.nargs)
 187             if '(' in self.args:
 188                 fmt = '(%s)' % fmt
 189             vals = ["repr(self.%s)" % name for name in self.argnames]
 190             vals = COMMA.join(vals)
 191             if self.nargs == 1:
 192                 vals = vals + ","
 193             print >> buf, '        return "%s(%s)" %% (%s)' % \
 194                   (self.name, fmt, vals)
 195         else:
 196             print >> buf, '        return "%s()"' % self.name
 197
 198 rx_init = re.compile('init\((.*)\):')
 199
 200 def parse_spec(file):
 201     classes = {}
 202     cur = None
 203     for line in fileinput.input(file):
 204         if line.strip().startswith('#'):
 205             continue
 206         mo = rx_init.search(line)
 207         if mo is None:
 208             if cur is None:
 209                 # a normal entry
 210                 try:
 211                     name, args = line.split(':')
 212                 except ValueError:
 213                     continue
 214                 classes[name] = NodeInfo(name, args)
 215                 cur = None
 216             else:
 217                 # some code for the __init__ method
 218                 cur.init.append(line)
 219         else:
 220             # some extra code for a Node's __init__ method
 221             name = mo.group(1)
 222             cur = classes[name]
 223     return sorted(classes.values(), key=lambda n: n.name)
 224
 225 def main():
 226     prologue, epilogue = load_boilerplate(sys.argv[-1])
 227     print prologue
 228     print
 229     classes = parse_spec(SPEC)
 230     for info in classes:
 231         print info.gen_source()
 232     print epilogue
 233
 234 if __name__ == "__main__":
 235     main()
 236     sys.exit(0)
 237
 238 ### PROLOGUE
 239 """Python abstract syntax node definitions
 240
 241 This file is automatically generated by Tools/compiler/astgen.py
 242 """
 243 from consts import CO_VARARGS, CO_VARKEYWORDS
 244
 245 def flatten(seq):
 246     l = []
 247     for elt in seq:
 248         t = type(elt)
 249         if t is tuple or t is list:
 250             for elt2 in flatten(elt):
 251                 l.append(elt2)
 252         else:
 253             l.append(elt)
 254     return l
 255
 256 def flatten_nodes(seq):
 257     return [n for n in flatten(seq) if isinstance(n, Node)]
 258
 259 nodes = {}
 260
 261 class Node:
 262     """Abstract base class for ast nodes."""
 263     def getChildren(self):
 264         pass # implemented by subclasses
 265     def __iter__(self):
 266         for n in self.getChildren():
 267             yield n
 268     def asList(self): # for backwards compatibility
 269         return self.getChildren()
 270     def getChildNodes(self):
 271         pass # implemented by subclasses
 272
 273 class EmptyNode(Node):
 274     pass
 275
 276 class Expression(Node):
 277     # Expression is an artificial node class to support "eval"
 278     nodes["expression"] = "Expression"
 279     def __init__(self, node):
 280         self.node = node
 281
 282     def getChildren(self):
 283         return self.node,
 284
 285     def getChildNodes(self):
 286         return self.node,
 287
 288     def __repr__(self):
 289         return "Expression(%s)" % (repr(self.node))
 290
 291 ### EPILOGUE
 292 for name, obj in globals().items():
 293     if isinstance(obj, type) and issubclass(obj, Node):
 294         nodes[name.lower()] = obj