tools/genmf/genmf.py

   1 #! @PYTHON@
   2 # -*- coding: iso-8859-1 -*-
   3 # Copyright © 2003-2008, The AROS Development Team. All rights reserved.
   4
   5 import sys, re, os, errno
   6
   7 if not len(sys.argv) in [2, 3, 4, 5] :
   8     print "Usage:",sys.argv[0],"tmplfile [inputfile outputfile]"
   9     print "Usage:",sys.argv[0],"tmplfile --usetmp [inputfile outputfile]"
  10     print "Usage:",sys.argv[0],"tmplfile --listfile filename"
  11     print "Usage:",sys.argv[0],"tmplfile --usetmp --listfile filename"
  12
  13 # A regular expression for the start of a template instantiation (ex. %build_module)
  14 re_tmplinst = re.compile('%([a-zA-Z0-9][a-zA-Z0-9_]*)(?=(?:\s|$))')
  15 # A regular expression for the argument specification during template instantiation
  16 # (ex. cflags=$(CFLAGS) or uselibs="amiga arosc")
  17 re_arg = re.compile('([a-zA-Z0-9][a-zA-Z0-9_]*)=([^\s"]+|".*?")?')
  18
  19 ##################################
  20 # Class and function definitions #
  21 ##################################
  22
  23 # Exception used throughout this program
  24 class GenmfException:
  25     def __init__(self, s):
  26         self.s = s
  27     def __str__(self):
  28         return self.s
  29
  30
  31 # Scan the given lines for template instantiations
  32 # Input params:
  33 # - lines: an array of strings
  34 # - templates: an assosiative array of objects of class template
  35 #
  36 # Returns:
  37 # - templrefs: an array of tuples with two elements. First element is the line number
  38 #   in the lines array, second is the result of the search for that line with
  39 #   re_tmplinst. Only templates which name are present in the templates argument
  40 #   will be added to this array.
  41 def generate_templrefs(lines, templates):
  42     templrefs = []
  43
  44     for lineno in range(len(lines)):
  45         line = lines[lineno]
  46         if len(line) == 0 or line[0] == "#":
  47             continue
  48
  49         m = re_tmplinst.search(line)
  50         if m and templates.has_key(m.group(1)) and not (m.start() > 0 and line[m.start()-1] == "#"):
  51             templrefs.append((lineno, m))
  52
  53     return templrefs
  54
  55
  56 # Write out the lines to a file with instantiating the templates present in the file
  57 # Input params:
  58 # - lines: The lines to write out
  59 # - templrefs: the instantiated templates present in these lines
  60 # - templates: the template definitions
  61 # - outfile: the file to write to
  62 #
  63 # This function does not return anything but raises a GenmfException when there are
  64 # problems
  65 def writelines(lines, templrefs, templates, outfile):
  66     start = 0
  67     for lineno, m in templrefs:
  68         if start<lineno:
  69             outfile.writelines(lines[start:lineno])
  70
  71         start = lineno + 1
  72         line = lines[lineno]
  73         while line[len(line)-2] == "\\" and start < len(lines):
  74             line = line[0:len(line)-2] + lines[start]
  75             start = start + 1
  76
  77         if m.group(1) == "common":
  78             template.hascommon = 1
  79
  80         try:
  81             templates[m.group(1)].write(outfile, m.group(1), line[m.end():].lstrip(), templates)
  82         except GenmfException, ge:
  83             raise GenmfException(("In instantiation of %s, line %d\n" % (m.group(1), lineno+1))+ge.s)
  84
  85     if start < len(lines):
  86         outfile.writelines(lines[start:len(lines)])
  87
  88
  89 # arg is a class that stores the specification of an argument in the template header.
  90 # The name of the arg is not stored in this class but this class is supposed to be
  91 # stored in an assosiative array with the names of the arg as indices. E.g
  92 # arg['cflags'] gives back an object of this class.
  93 # It has the following members:
  94 # - ismulti: boolean to indicate if it has the /M definition
  95 # - isneeded: boolean to indicate if it has the /A definition
  96 # - used: boolean to indicate if a value is used in the body of a template
  97 # - default: default value when the argument is not given during a template
  98 #   instantiation
  99 # - value: The value this argument has during a template instantiation is stored
 100 #   here
 101 class arg:
 102     # Specification can end with /A or /M
 103     re_mode = re.compile('/(A|M)')
 104
 105     # You create this object with giving it the default value
 106     def __init__(self, default=None):
 107         self.ismulti = 0
 108         self.isneeded = 0
 109         self.used = 0
 110
 111         while default and len(default)>1:
 112             m = arg.re_mode.match(default[len(default)-2:])
 113             if not m:
 114                 break
 115             if m.group(1) == "M":
 116                 self.ismulti = 1
 117             elif m.group(1) == "A":
 118                 self.isneeded = 1
 119             else:
 120                 sys.exit('Internal error: Unknown match')
 121
 122             default = default[:len(default)-2]
 123
 124         if default and default[0] == '"':
 125             default = default[1:len(default)-1]
 126
 127         self.default = default
 128         # The value field will get the value passed to the argument when the template is used
 129         self.value = None
 130
 131
 132 # template is a class to store the whole definition of a genmf template
 133 # Members:
 134 # - name: name of the template
 135 # - args: an associative array of the arguments of this template
 136 # - body: an array of strings with the body of the template
 137 # - multiarg: contains the arg with /M if it is present
 138 # - used: Is this template already used; used to check for recursive calling
 139 #   of a template
 140 # - linerefs: an array to indicate the genmf variables used in the body of the
 141 #   template. This is generated with the generate_linerefs method of this class
 142 # - templrefs: an array to indicate the templates used in the body of the template.
 143 #   This is generated with the generate_templrefs function of this class.
 144 class template:
 145     re_arginst = re.compile('([a-zA-Z0-9-_]*)%\(([a-zA-Z0-9][a-zA-Z0-9_]*)\)([a-zA-Z0-9-_]*)')
 146     hascommon = 0
 147
 148     # Generate a template
 149     # Input params:
 150     # - name: name of the template
 151     # - args: an assosiative array of the arguments defined for this template
 152     # - body: an array of the template with the bodylines of the template
 153     def __init__(self, name, args, body):
 154         self.name = name
 155         self.args = args
 156         self.body = body
 157         self.multiarg = None
 158         self.used = 0
 159         self.linerefs = None
 160         self.templrefs = None
 161
 162         for argname, argbody in args.items():
 163             if argbody.ismulti:
 164                 if self.multiarg:
 165                     sys.exit('A template can have only one main (/M) argument')
 166                 self.multiarg = argbody
 167
 168     # Generate the references for the genmf variable used in this template
 169     # This function will return an assositive array of tuples with linerefs[lineno]
 170     # an array of tuples named argrefs with the tuple of the form (argbody, start, stop, prefix, suffix)
 171     # with argbody an object of class arg, start and stop the start and end of this variable
 172     # in the string of this line.
 173     def generate_linerefs(self):
 174         lineno = 0
 175         linerefs = {}
 176         while lineno < len(self.body):
 177             argrefs = []
 178             for m in template.re_arginst.finditer(self.body[lineno]):
 179                 if self.args.has_key(m.group(2)):
 180                     argbody = self.args[m.group(2)]
 181                     argrefs.append((argbody, m.start(), m.end(),m.group(1),m.group(3)))
 182                     argbody.used = 1
 183
 184             if len(argrefs) > 0:
 185                 linerefs[lineno] = argrefs
 186
 187             lineno = lineno+1
 188         self.linerefs = linerefs
 189
 190         for argname, argbody in self.args.items():
 191             if not argbody.used:
 192                 sys.stderr.write("Warning: template '%s': unused argument '%s'\n" % (self.name, argname))
 193
 194
 195     # Write out the body of the template
 196     def write(self, outfile, name, line, templates):
 197         if self.used:
 198             raise GenmfException("Template '%s' called recursively" % name)
 199         self.used = 1
 200
 201         # Reading arguments of the template
 202         argno = 0
 203         while len(line) > 0:
 204             m = re_arg.match(line)
 205             if m and self.args.has_key(m.group(1)):
 206                 value = m.group(2)
 207                 if value == None:
 208                     #sys.stderr.write("Arg:"+m.group(1)+" Value: None Line:"+line+"\n")
 209                     self.args[m.group(1)].value = ''
 210                 else:
 211                     #sys.stderr.write("Arg:"+m.group(1)+" Value:"+m.group(2)+" Line:"+line+"\n")
 212                     if len(value)>0 and value[0] == '"':
 213                         value = value[1:len(value)-1]
 214                     self.args[m.group(1)].value = value
 215                 line = line[m.end():].lstrip()
 216             elif self.multiarg:
 217                 self.multiarg.value = line[:len(line)-1]
 218                 line = ''
 219             else:
 220                 raise GenmfException('Syntax error in arguments: '+line)
 221
 222         if self.linerefs == None:
 223             self.generate_linerefs()
 224             self.templrefs = generate_templrefs(self.body, templates)
 225
 226         for argname, argbody in self.args.items():
 227             if argbody.isneeded and argbody.value == None:
 228                 raise GenmfException('Arg "%s" not specified but should have been' % argname)
 229
 230         text = self.body[:]
 231
 232         for lineno, argrefs in self.linerefs.items():
 233             line = text[lineno]
 234
 235             pos=0
 236             lineout = ''
 237             for argref in argrefs:
 238                 if argref[1] > pos:
 239                     lineout = lineout + line[pos:argref[1]]
 240
 241                 linevals=[]
 242                 if not argref[0].value == None:
 243                     for val in argref[0].value.split():
 244                         linevals.append(argref[3] + val + argref[4])
 245                 elif argref[0].default:
 246                     for val in argref[0].default.split():
 247                         linevals.append(argref[3] + val + argref[4])
 248
 249                 if len(linevals) == 0:
 250                     lineout += argref[3] + argref[4]
 251                 else:
 252                     lineout += " ".join(linevals)
 253
 254                 pos = argref[2]
 255
 256             if pos < len(line):
 257                 lineout = lineout + line[pos:]
 258
 259             text[lineno] = lineout
 260
 261         writelines(text, self.templrefs, templates, outfile)
 262         #outfile.write('\n')
 263
 264         for argname, argbody in self.args.items():
 265             argbody.value = None
 266         self.used = 0
 267
 268
 269
 270 # Read in the definition of the genmf templates from the given filename
 271 # Return an assosiative array of the templates present in this file.
 272 def read_templates(filename):
 273     try:
 274         infile = open(filename)
 275     except:
 276         print "Error reading template file: "+filename
 277
 278     re_name = re.compile('[a-zA-Z0-9][a-zA-Z0-9_]*(?=(?:\s|$))')
 279     re_openstring = re.compile('[^\s"]*"[^"]*$')
 280     re_define = re.compile('%define(?=\s)')
 281
 282     lines = infile.readlines()
 283     lineno = 0
 284     templates = {}
 285     while lineno < len(lines):
 286         line = lines[lineno]
 287         if re_define.match(line):
 288             while line[len(line)-2] == "\\" and lineno < len(lines):
 289                 lineno = lineno + 1
 290                 line = line[0:len(line)-2] + lines[lineno]
 291
 292             line = line[7:].strip()
 293
 294             m = re_name.match(line)
 295             if not m:
 296                 sys.exit("%s:%d:Error in syntax of template name" % (filename, lineno+1))
 297             tmplname = m.group(0)
 298             line = line[m.end():].lstrip()
 299
 300             args = {}
 301             while len(line) > 0:
 302                 m = re_arg.match(line)
 303                 if not m:
 304                     sys.exit("%s:%d:Error in syntax of argument %d Line: %s" % (filename, lineno+1, len(args)+1, line))
 305                 args[m.group(1)] = arg(m.group(2))
 306
 307                 line = line[m.end():].lstrip()
 308
 309             #print "Line: %d Template: %s" % (lineno+1, tmplname)
 310
 311             lineno = lineno+1
 312             line = lines[lineno]
 313             bodystart = lineno
 314             while lineno < len(lines) and line[0:4] <> "%end":
 315                 lineno = lineno+1
 316                 line = lines[lineno]
 317
 318             if lineno == len(lines):
 319                 sys.exit('%s:End of file reached in a template definition' % filename)
 320
 321             templates[tmplname] = template(tmplname, args, lines[bodystart:lineno])
 322
 323         lineno = lineno+1
 324
 325     return templates
 326
 327
 328 ################
 329 # Main program #
 330 ################
 331
 332 argv = []
 333 i = 0
 334 listfile = None
 335 progcount = 0
 336 usetemp = 0
 337 argin = 2
 338 while i < len(sys.argv):
 339     if sys.argv[i] == "--listfile":
 340         listfile = sys.argv[i+1]
 341         i = i + 2
 342     elif sys.argv[i] == "--usetmp":
 343        i = i + 1
 344        argin = 3
 345        usetemp = 1
 346     else:
 347         argv.append(sys.argv[i])
 348         i = i + 1
 349
 350 #sys.stderr.write("Reading templates\n")
 351 templates = read_templates(argv[1])
 352 #sys.stderr.write("Read %d templates\n" % len(templates))
 353
 354 if listfile == None:
 355     # Read one input file and write out one outputfile
 356     if len(sys.argv) == argin:
 357         lines = sys.stdin.readlines()
 358     else:
 359         infile = open(sys.argv[argin], "r")
 360         lines = infile.readlines()
 361         infile.close()
 362
 363     if len(sys.argv) == argin:
 364         outfile = sys.stdout
 365         closeout = 0
 366     else:
 367         if usetemp:
 368             outfile = open(sys.argv[argin + 1]+"tmp", "w")
 369         else:
 370             outfile = open(sys.argv[argin + 1], "w")
 371         closeout = 1
 372
 373     try:
 374         writelines(lines, generate_templrefs(lines, templates), templates, outfile)
 375     except GenmfException, ge:
 376         s = ge.s
 377         if len(sys.argv) == argin + 2:
 378             s = sys.argv[argin + 1]+":"+s
 379             sys.exit(s+"\n")
 380
 381     # If %common was not present in the file write it out at the end of the file
 382     if not template.hascommon:
 383         outfile.write("\n")
 384         if templates.has_key("common"):
 385             templates["common"].write(outfile, "common", "", templates)
 386
 387     if closeout:
 388         outfile.close()
 389         if usetemp:
 390             if os.path.isfile(files[1]):
 391                 os.remove(sys.argv[argin + 1])
 392             os.rename(sys.argv[argin + 1]+"tmp", sys.argv[argin + 1])
 393
 394 else:
 395     # When a listfile is specified each line in this listfile is of the form
 396     # inputfile outputfile
 397     # Apply the instantiation of the templates to all these files listed there
 398     infile = open(listfile, "r")
 399     filelist = infile.readlines()
 400     infile.close()
 401
 402     sys.stderr.write('Regenerating %d files\n' % (len(filelist)))
 403
 404     for fileno in range(len(filelist)):
 405         files = filelist[fileno].split()
 406         if len(files) <> 2:
 407             sys.exit('%s:%d: Syntax error: %s' % (listfile, fileno+1, filelist[fileno]))
 408
 409         progcount = progcount + 1
 410         if progcount == 20:
 411             progcount = 0
 412             sys.stderr.write('.')
 413             sys.stderr.flush()
 414
 415         infile = open(files[0], "r")
 416         lines = infile.readlines()
 417         infile.close()
 418
 419         try:
 420             # os.makedirs will also create all the parent directories
 421             os.makedirs(os.path.dirname(files[1]))
 422         except OSError, err:
 423             # Do nothing ..
 424             s = err.errno
 425
 426         if usetemp:
 427             outfile = open(files[1]+"tmp", "w")
 428         else:
 429             outfile = open(files[1], "w")
 430
 431         template.hascommon = 0
 432
 433         try:
 434             writelines(lines, generate_templrefs(lines, templates), templates, outfile)
 435         except GenmfException, ge:
 436             s = ge.s
 437             if len(sys.argv) == argin + 2:
 438                 s = files[0]+":"+s
 439                 sys.exit(s+"\n")
 440
 441         if not template.hascommon:
 442             outfile.write("\n")
 443             if templates.has_key("common"):
 444                 templates["common"].write(outfile, "common", "", templates)
 445
 446         outfile.close()
 447         if usetemp:
 448             if os.path.isfile(files[1]):
 449                 os.remove(files[1])
 450             os.rename(files[1]+"tmp", files[1])
 451
 452     sys.stderr.write('\n')