scripts/compiler.py

   1 #!/usr/bin/python
   2
   3 import sys
   4 import struct
   5 import os
   6
   7 option_ptrsize = struct.calcsize("P")
   8 option_intsize = struct.calcsize("l")
   9 option_floatsize = struct.calcsize("f")
  10 option_inttype = "long"
  11 option_floattype = "float"
  12
  13 class node:
  14         def __init__(self):
  15                 self.values = []
  16                 self.children = []
  17                 self.parent = 0
  18
  19         def name(self):
  20                 if len(self.values):
  21                         return self.values[0]
  22                 return ""
  23
  24         def debug_print(self, level):
  25                 print ("  "*level) + " ".join(self.values),
  26                 if len(self.children):
  27                         print "{"
  28                         for c in self.children:
  29                                 c.debug_print(level+1)
  30                         print ("  "*level)+"}"
  31                 else:
  32                         print ""
  33
  34         def debug_root(self):
  35                 for c in self.children:
  36                         c.debug_print(0)
  37
  38         # TODO: should return list of items in the tree,
  39         def gather(self, str):
  40                 def recurse(parts, path, node):
  41                         if not len(parts):
  42                                 r = {}
  43                                 path = path + "." + node.values[0]
  44                                 r = [node]
  45                                 #print "found", path
  46                                 return r
  47
  48                         l = []
  49                         for c in node.children:
  50                                 if parts[0] == "*" or c.values[0] == parts[0]:
  51                                         if len(node.values):
  52                                                 if len(path):
  53                                                         l += recurse(parts[1:], path+"."+node.values[0], c)
  54                                                 else:
  55                                                         l += recurse(parts[1:], node.values[0], c)
  56                                         else:
  57                                                 l += recurse(parts[1:], path, c)
  58                         return l
  59
  60                 parts = str.split(".")
  61                 return recurse(parts, "", self)
  62
  63         def find_node(self, str):
  64                 parts = str.split(".")
  65                 node = self
  66                 for part in parts:
  67                         if len(part) == 0:
  68                                 continue
  69                         if part == "parent":
  70                                 node = node.parent
  71                         else:
  72                                 found = 0
  73                                 for c in node.children:
  74                                         if part == c.values[0]:
  75                                                 node = c
  76                                                 found = 1
  77                                                 break
  78
  79                 if node == self:
  80                         return
  81                 return node
  82
  83         def get_single(self, str):
  84                 parts = str.split("@")
  85                 index = -1
  86                 if len(parts) == 2:
  87                         index = int(parts[1])
  88
  89                 node = self
  90                 if len(parts[0]):
  91                         node = self.find_node(parts[0])
  92
  93                 if not node:
  94                         print "failed to get", str
  95                         return Null
  96
  97                 if index == -1:
  98                         return node.get_path()[1:]
  99                 return node.values[index]
 100
 101         def get_path(self):
 102                 if self.parent == 0:
 103                         return ""
 104                 return self.parent.get_path() + "." + self.values[0]
 105
 106         def get_single_name(self, str):
 107                 return self.get_path()[1:] + "." + str
 108
 109 class parser:
 110         lines = []
 111
 112         def parse_node(self, this_node):
 113                 while len(self.lines):
 114                         line = self.lines.pop(0)                # grab line
 115
 116                         fields = line.strip().split() # TODO: improve this to handle strings with spaces
 117                         if not len(fields):
 118                                 continue
 119
 120                         if fields[-1] == '{':
 121                                 new_node = node()
 122                                 new_node.parent = this_node
 123                                 new_node.values = fields[:-1]
 124                                 this_node.children += [new_node]
 125                                 self.parse_node(new_node)
 126                         elif fields[-1] == '}':
 127                                 break
 128                         else:
 129                                 new_node = node()
 130                                 new_node.parent = this_node
 131                                 new_node.values = fields
 132                                 this_node.children += [new_node]
 133
 134         def parse_file(self, filename):
 135                 self.lines = file(filename).readlines()
 136                 n = node()
 137                 self.parse_node(n)
 138                 return n
 139
 140 def parse_file(filename):
 141         return parser().parse_file(filename)
 142
 143 class pointer:
 144         def __init__(self, index, target):
 145                 self.index = index
 146                 self.target = target
 147
 148 class data_constructor:
 149         def __init__(self):
 150                 self.data = ""
 151                 self.trans = 0
 152                 self.pointers = []
 153                 self.targets = {}
 154                 self.enums = {}
 155
 156         def get_type(self, s):
 157                 return self.trans.types[s]
 158
 159         def allocate(self, size):
 160                 index = len(self.data)
 161                 self.data += "\0"*size
 162                 return index
 163
 164         def add_pointer(self, index, target):
 165                 self.pointers += [pointer(index, target)]
 166
 167         def add_enum(self, name, value):
 168                 self.enums[name] = value
 169
 170         def get_enum_value(self, name):
 171                 if not name in self.enums:
 172                         print "ERROR: couldn't find enum '%s'" % (name)
 173                 return self.enums[name]
 174
 175         def add_target(self, target, index):
 176                 # TODO: warn about duplicates
 177                 #print "add_target(target='%s' index=%d)" % (target, index)
 178                 self.targets[target] = index
 179
 180         def write(self, index, size, data):
 181                 try:
 182                         self.data = self.data[:index] + data + self.data[index+size:]
 183                 except:
 184                         print "write error:"
 185                         print "\tself.data =", self.data
 186                         print "\tdata =", data
 187
 188         def patch_pointers(self):
 189                 for p in self.pointers:
 190                         if p.target in self.targets:
 191                                 i = self.targets[p.target]
 192                                 #print "ptr @ %d -> %s -> %d" % (p.index, p.target, i)
 193                                 data = struct.pack("P", i)
 194                                 self.write(p.index, len(data), data)
 195                         else:
 196                                 print "ERROR: couldn't find target '%s' for pointer at %d" % (p.target, p.index)
 197
 198 class type:
 199         def __init__(self):
 200                 self.name = ""
 201
 202         def size(self):
 203                 pass
 204
 205 class structure:
 206         def __init__(self):
 207                 self.name = ""
 208                 self.members = []
 209
 210         def size(self):
 211                 s = 0
 212                 for m in self.members:
 213                         s += m.size()
 214                 return s
 215
 216         def emit_header_code(self, out):
 217                 print >>out, "struct", self.name
 218                 print >>out, "{"
 219                 for m in self.members:
 220                         for l in m.get_code():
 221                                 print >>out, "\t" + l
 222                 print >>out, "};"
 223                 print >>out, ""
 224
 225         def emit_source_code(self, out):
 226                 print >>out, "static void patch_ptr_%s(%s *self, char *base)" % (self.name, self.name)
 227                 print >>out, "{"
 228                 for m in self.members:
 229                         for l in m.get_patch_code("self", "base"):
 230                                 print >>out, "\t" + l
 231                 print >>out, "}"
 232                 print >>out, ""
 233
 234         def emit_data(self, cons, index, src_data):
 235                 #print self.name+":"
 236                 member_index = index
 237                 for m in self.members:
 238                         #print "\t" + m.name
 239                         m.emit_data(cons, member_index, src_data)
 240                         member_index += m.size()
 241
 242 class variable:
 243         def __init__(self):
 244                 self.expr = ""
 245                 self.type = ""
 246                 self.subtype = ""
 247
 248         def get_code(self):
 249                 return []
 250
 251         def get_patch_code(self, ptrname, basename):
 252                 return []
 253
 254         def emit_data(self, cons, index, src_data):
 255                 pass
 256
 257 class variable_int(variable):
 258         def get_code(self):
 259                 return ["%s %s;" % (option_inttype, self.name)]
 260         def size(self):
 261                 return option_intsize
 262         def emit_data(self, cons, index, src_data):
 263                 try:
 264                         value = int(self.expr)
 265                 except:
 266                         value = int(src_data.get_single(self.expr))
 267                 #print "int", self.name, "=", value, "@", index
 268                 data = struct.pack("l", value)
 269                 cons.write(index, len(data), data)
 270
 271 class variable_float(variable):
 272         def get_code(self):
 273                 return ["%s %s;" % (option_floattype, self.name)]
 274         def size(self):
 275                 return option_floatsize
 276         def emit_data(self, cons, index, src_data):
 277                 try:
 278                         value = float(self.expr)
 279                 except:
 280                         value = float(src_data.get_single(self.expr))
 281                 #print "int", self.name, "=", value, "@", index
 282                 data = struct.pack("f", value)
 283                 cons.write(index, len(data), data)
 284
 285 class variable_string(variable):
 286         def get_code(self):
 287                 return ["char *%s;" % (self.name)]
 288         def get_patch_code(self, ptrname, basename):
 289                 return ["patch_ptr((char **)&(%s->%s), %s);" % (ptrname, self.name, basename)]
 290         def size(self):
 291                 return option_ptrsize
 292         def emit_data(self, cons, index, src_data):
 293                 string = src_data.get_single(self.expr)
 294                 string = string.strip()[1:-1] # skip " and "
 295
 296                 string_index = cons.allocate(len(string)+1)
 297                 cons.write(string_index, len(string), string)
 298
 299                 data = struct.pack("P", string_index) # TODO: solve this
 300                 cons.write(index, len(data), data)
 301
 302 class variable_ptr(variable):
 303         def get_code(self):
 304                 return ["%s *%s;" % (self.subtype, self.name)]
 305         def get_patch_code(self, ptrname, basename):
 306                 return ["patch_ptr((char**)&(%s->%s), %s);" % (ptrname, self.name, basename)]
 307         def size(self):
 308                 return option_ptrsize
 309         def emit_data(self, cons, index, src_data):
 310                 target = src_data.get_single(self.expr)
 311                 cons.add_pointer(index, target)
 312
 313 class variable_enum(variable):
 314         def get_code(self):
 315                 return ["long *%s;" % (self.name)]
 316         def size(self):
 317                 return option_intsize
 318         def emit_data(self, cons, index, src_data):
 319                 target = src_data.get_single(self.expr)
 320                 data = struct.pack("l", cons.get_enum_value(target))
 321                 cons.write(index, len(data), data)
 322
 323 class variable_instance(variable):
 324         def get_code(self):
 325                 return ["%s %s;" % (self.subtype, self.name)]
 326         def get_patch_code(self, ptrname, basename):
 327                 return ["patch_ptr_%s(&(%s->%s), %s);" % (self.subtype, ptrname, self.name, basename)]
 328         def size(self):
 329                 return self.translator.types[self.subtype].size()
 330         def emit_data(self, cons, index, src_data):
 331                 target = src_data.find_node(self.expr)
 332                 translator.types[self.subtype].emit_data(cons, index, target)
 333                 #target =
 334                 #cons.add_pointer(index, target)
 335
 336 class variable_array(variable):
 337         def get_code(self):
 338                 return ["long num_%s;" % self.name,
 339                         "%s *%s;" % (self.subtype, self.name)]
 340         def get_patch_code(self, ptrname, baseptr):
 341                 code = []
 342                 code += ["patch_ptr((char**)&(%s->%s), %s);" % (ptrname, self.name, baseptr)]
 343                 code += ["for(int i = 0; i < %s->num_%s; i++)" % (ptrname, self.name)]
 344                 code += ["\tpatch_ptr_%s(%s->%s+i, %s);" % (self.subtype, ptrname, self.name, baseptr)]
 345                 return code
 346         def emit_data(self, cons, index, src_data):
 347                 array_data = src_data.gather(self.expr)
 348                 array_type = cons.get_type(self.subtype)
 349                 size = array_type.size()*len(array_data)
 350
 351                 #print "packing array", self.name
 352                 #print "\ttype =", array_type.name
 353                 #print "\tsize =", array_type.size()
 354                 array_index = cons.allocate(size)
 355                 data = struct.pack("lP", len(array_data), array_index) # TODO: solve this
 356                 cons.write(index, len(data), data)
 357
 358                 member_index = array_index
 359                 for node in array_data:
 360                         cons.add_target(node.get_path()[1:], member_index)
 361                         array_type.emit_data(cons, member_index, node)
 362                         member_index += array_type.size()
 363                         #print "array", member_index
 364
 365         def size(self):
 366                 return option_ptrsize+option_intsize
 367
 368 class const_arrayint:
 369         def __init__(self):
 370                 self.name = ""
 371                 self.values = []
 372
 373         def emit_header_code(self, out):
 374                 print >>out, "enum"
 375                 print >>out, "{"
 376                 for i in xrange(0, len(self.values)):
 377                         print >>out, "\t%s_%s = %d," % (self.name.upper(), self.values[i].upper(), i)
 378
 379                 print >>out, "\tNUM_%sS = %d" % (self.name.upper(), len(self.values))
 380                 print >>out, "};"
 381                 print >>out, ""
 382
 383 class translator:
 384         def __init__(self):
 385                 self.types = {}
 386                 self.structs = []
 387                 self.constants = []
 388                 self.data = 0
 389                 self.srcdata = 0
 390
 391                 self.types["int"] = variable_int()
 392                 self.types["float"] = variable_float()
 393                 self.types["string"] = variable_string()
 394                 self.types["ptr"] = variable_ptr()
 395                 self.types["array"] = variable_array()
 396
 397         def parse_variable(self, node):
 398                 if len(node.values) != 4:
 399                         print node.values
 400                         raise "error parsing variable"
 401
 402                 type = node.values[0]
 403                 subtype = ""
 404                 if type == "int":
 405                         v = variable_int()
 406                 elif type == "enum":
 407                         v = variable_enum()
 408                 elif type == "float":
 409                         v = variable_float()
 410                 elif type == "string":
 411                         v = variable_string()
 412                 else:
 413                         # complex type
 414                         parts = type.split(":")
 415                         if len(parts) != 2:
 416                                 raise "can't emit code for variable %s of type %s" % (self.name, self.type)
 417                         elif parts[0] == "ptr":
 418                                 subtype = parts[1]
 419                                 v = variable_ptr()
 420                         elif parts[0] == "instance":
 421                                 subtype = parts[1]
 422                                 v = variable_instance()
 423                         elif parts[0] == "array":
 424                                 subtype = parts[1]
 425                                 v = variable_array()
 426                         else:
 427                                 raise "can't emit code for variable %s of type %s" % (self.name, self.type)
 428
 429                 v.translator = self
 430                 v.type = node.values[0]
 431                 v.subtype = subtype
 432                 v.name = node.values[1]
 433                 assignment = node.values[2]
 434                 v.expr = node.values[3]
 435                 if assignment != "=":
 436                         raise "error parsing variable. expected ="
 437                 return v
 438
 439         def parse_struct(self, node):
 440                 if len(node.values) != 2:
 441                         raise "error parsing struct"
 442                 s = structure()
 443                 s.name = node.values[1]
 444
 445                 for statement in node.children:
 446                         s.members += [self.parse_variable(statement)]
 447                 return s
 448
 449         def parse_constant(self, node):
 450                 if len(node.values) != 5:
 451                         print node.values
 452                         raise "error parsing constant"
 453
 454                 type = node.values[1]
 455                 name = node.values[2]
 456                 assignment = node.values[3]
 457                 expression = node.values[4]
 458
 459                 if assignment != "=":
 460                         print node.values
 461                         raise "error parsing constant"
 462
 463                 ints = const_arrayint()
 464                 ints.name = name
 465
 466                 items = self.srcdata.gather(expression)
 467                 for c in items:
 468                         ints.values += [c.name()]
 469                 self.constants += [ints]
 470
 471         def parse(self, script, srcdata):
 472                 self.srcdata = srcdata
 473                 for statement in script.children:
 474                         if statement.values[0] == "struct":
 475                                 s = self.parse_struct(statement)
 476                                 self.structs += [s]
 477                                 self.types[s.name] = s
 478                         elif statement.values[0] == "const":
 479                                 self.parse_constant(statement)
 480                         else:
 481                                 raise "unknown statement:" + statement
 482
 483         def emit_header_code(self, out):
 484                 for c in self.constants:
 485                         c.emit_header_code(out)
 486
 487                 for s in self.structs:
 488                         s.emit_header_code(out)
 489                 print >>out, ""
 490                 print >>out, "struct data_container *load_data_from_file(const char *filename);"
 491                 print >>out, "struct data_container *load_data_from_memory(unsigned char *filename);"
 492                 print >>out, ""
 493
 494
 495         def emit_source_code(self, out, header_filename):
 496                 print >>out, '''
 497
 498 #include "%s"
 499 #include <stdio.h>
 500 #include <stdlib.h>
 501
 502 static void patch_ptr(char **ptr, char *base)
 503 {
 504         *ptr = base+(size_t)(*ptr);
 505 }
 506 ''' % header_filename
 507
 508                 for s in self.structs:
 509                         s.emit_source_code(out)
 510                 print >>out, '''
 511
 512 data_container *load_data_from_memory(unsigned char *mem)
 513 {
 514         if(mem[0] != sizeof(void*))
 515                 return 0;
 516         if(mem[1] != sizeof(long))
 517                 return 0;
 518         if(mem[2] != sizeof(float))
 519                 return 0;
 520
 521         /* patch all pointers */
 522         data_container *con = (data_container*)(mem + 4);
 523         patch_ptr_data_container(con, (char *)con);
 524         return con;
 525 }
 526
 527 data_container *load_data_from_file(const char *filename)
 528 {
 529         unsigned char *data = 0;
 530         int size;
 531
 532         /* open file */
 533         FILE *f = fopen(filename, "rb");
 534
 535         /* get size */
 536         fseek(f, 0, SEEK_END);
 537         size = ftell(f);
 538         fseek(f, 0, SEEK_SET);
 539
 540         /* allocate, read data and close file */
 541         data = (unsigned char *)malloc(size);
 542         fread(data, 1, size, f);
 543         fclose(f);
 544
 545         return load_data_from_memory(data);
 546 }
 547
 548 '''
 549
 550         def emit_data(self):
 551                 for s in self.structs:
 552                         if s.name == "data_container":
 553                                 #print "found data_container"
 554                                 cons = data_constructor()
 555                                 cons.trans = self
 556                                 i = cons.allocate(s.size())
 557                                 s.emit_data(cons, i, self.srcdata)
 558                                 cons.patch_pointers()
 559                                 header = struct.pack("bbbb", option_ptrsize, option_intsize, option_floatsize, 0)
 560                                 return header + cons.data
 561
 562 def create_translator(script, srcdata):
 563         t = translator()
 564         t.parse(script, srcdata)
 565         return t
 566
 567 def validate(script, validator):
 568         def validate_values(values, check):
 569                 if not len(check) or check[0] == "*":
 570                         print "too many values"
 571                         return
 572                 p = check[0].split(":")
 573                 type = p[0]
 574                 name = p[1]
 575
 576                 # TODO: check type and stuff
 577
 578                 # recurse
 579                 if len(values) > 1:
 580                         if not len(check):
 581                                 print "unexpected value"
 582                         validate_values(values[1:], check[1:])
 583                 else:
 584                         if len(check) > 1 and check[1] != "*":
 585                                 print "to few values"
 586
 587         if len(script.values):
 588                 validate_values(script.values, validator.values)
 589
 590         for child in script.children:
 591                 tag = child.values[0]
 592                 n = validator.find_node("tag:"+tag)
 593                 if not n:
 594                         found = 0
 595                         for vc in validator.children:
 596                                 if "ident:" in vc.values[0]:
 597                                         validate(child, vc)
 598                                         print vc.values[0]
 599                                         found = 1
 600                                         break
 601
 602                         if not found:
 603                                 print "error:", tag, "not found"
 604                 else:
 605                         print "tag:"+tag
 606                         validate(child, n)
 607
 608 input_filename = sys.argv[1]
 609 script_filename = sys.argv[2]
 610
 611 output_filename = 0
 612 coutput_filename = 0
 613 header_filename = 0
 614 source_filename = 0
 615 sheader_filename = 0
 616
 617 if sys.argv[3] == '-h':
 618         header_filename = sys.argv[4]
 619 elif sys.argv[3] == '-s':
 620         source_filename = sys.argv[4]
 621         sheader_filename = sys.argv[5]
 622 elif sys.argv[3] == '-d':
 623         output_filename = sys.argv[4]
 624 elif sys.argv[3] == '-c':
 625         coutput_filename = sys.argv[4]
 626
 627 srcdata = parse_file(input_filename)
 628 script = parse_file(script_filename)
 629
 630 translator = create_translator(script, srcdata)
 631
 632 if header_filename:
 633         translator.emit_header_code(file(header_filename, "w"))
 634 if source_filename:
 635         translator.emit_source_code(file(source_filename, "w"), os.path.basename(sheader_filename))
 636
 637 if output_filename:
 638         rawdata = translator.emit_data()
 639         file(output_filename, "wb").write(rawdata)
 640 if coutput_filename:
 641         i = 0
 642         rawdata = translator.emit_data()
 643         f = file(coutput_filename, "w")
 644
 645         print >>f,"unsigned char internal_data[] = {"
 646         print >>f,str(ord(rawdata[0])),
 647         for d in rawdata[1:]:
 648             s = ","+str(ord(d))
 649             print >>f,s,
 650             i += len(s)+1
 651
 652             if i >= 70:
 653                 print >>f,""
 654                 i = 0
 655         print >>f,""
 656         print >>f,"};"
 657         print >>f,""
 658         f.close()