tools/genmf/genmf.py

   1 #! @PYTHON@
   2 # -*- coding: iso-8859-1 -*-
   3 # Copyright © 2003-2004, The AROS Development Team. All rights reserved.
   4
   5 import sys, re
   6
   7
   8 if not len(sys.argv) in [2, 4] :
   9     print "Usage:",sys.argv[0],"tmplfile [inputfile outputfile]"
  10     print "Usage:",sys.argv[0],"tmplfile --listfile filename"
  11
  12 # A regular expression for the start of a template instantiation (ex. %build_module)
  13 re_tmplinst = re.compile('%([a-zA-Z0-9][a-zA-Z0-9_]*)(?=(?:\s|$))')
  14 # A regular expression for the argument specification during template instantiation
  15 # (ex. cflags=$(CFLAGS) or uselibs="amiga arosc")
  16 re_arg = re.compile('([a-zA-Z0-9][a-zA-Z0-9_]*)=([^\s"]+|".*?")?')
  17
  18 ##################################
  19 # Class and function definitions #
  20 ##################################
  21
  22 # Exception used throughout this program
  23 class GenmfException:
  24     def __init__(self, s):
  25         self.s = s
  26     def __str__(self):
  27         return self.s
  28
  29
  30 # Scan the given lines for template instantiations
  31 # Input params:
  32 # - lines: an array of strings
  33 # - templates: an assosiative array of objects of class template
  34 #
  35 # Returns:
  36 # - templrefs: an array of tuples with two elems. First element is the lineno
  37 #   in the lines array, second is the results of the search for that line with
  38 #   re_tmplinst. Only templates which name are present in the templates argument
  39 #   will be added to this array.
  40 def generate_templrefs(lines, templates):
  41     templrefs = []
  42
  43     for lineno in range(len(lines)):
  44         line = lines[lineno]
  45         if len(line) == 0 or line[0] == "#":
  46             continue
  47
  48         m = re_tmplinst.search(line)
  49         if m and templates.has_key(m.group(1)) and not (m.start() > 0 and line[m.start()-1] == "#"):
  50             templrefs.append((lineno, m))
  51
  52     return templrefs
  53
  54
  55 # Write out the lines to a file with instantiating the templates present in the file
  56 # Input params:
  57 # - lines: The lines to write out
  58 # - templrefs: the instantiated templates present in these lines
  59 # - templates: the template definitions
  60 # - outfile: the file to write to
  61 #
  62 # This function does not return anything but raises a GenmfException when there are
  63 # problems
  64 def writelines(lines, templrefs, templates, outfile):
  65     start = 0
  66     for lineno, m in templrefs:
  67         if start<lineno:
  68             outfile.writelines(lines[start:lineno])
  69
  70         start = lineno + 1
  71         line = lines[lineno]
  72         while line[len(line)-2] == "\\" and start < len(lines):
  73             line = line[0:len(line)-2] + lines[start]
  74             start = start + 1
  75
  76         if m.group(1) == "common":
  77             template.hascommon = 1
  78
  79         try:
  80             templates[m.group(1)].write(outfile, m.group(1), line[m.end():].lstrip(), templates)
  81         except GenmfException, ge:
  82             raise GenmfException(("In instantiation of %s, line %d\n" % (m.group(1), lineno+1))+ge.s)
  83
  84     if start < len(lines):
  85         outfile.writelines(lines[start:len(lines)])
  86
  87
  88 # arg is q class that stores the specification of an argument in the template header
  89 # The of the arg is not stored in this class but this class is supposed to be
  90 # stored in an assosiative array with the names of the arg as indices. E.g
  91 # arg['cflags'] gives back an object of this class.
  92 # It has the following members:
  93 # - ismain: boolean to indicate if it has the /M definition
  94 # - isneeded: boolean to indicate if it has the /A definition
  95 # - used: boolean to indicate if a value is used in the body of a template
  96 # - default: default value when the argument is not given during a template
  97 #   instantiation
  98 # - value: The value this argument has during a template instantiation is stored
  99 #   here
 100 class arg:
 101     # Specification can end with /A or /M
 102     re_mode = re.compile('/(A|M)')
 103
 104     # You create this object with giving it the default value
 105     def __init__(self, default=None):
 106         self.ismain = 0
 107         self.isneeded = 0
 108         self.used = 0
 109
 110         while default and len(default)>1:
 111             m = arg.re_mode.match(default[len(default)-2:])
 112             if not m:
 113                 break
 114             if m.group(1) == "M":
 115                 self.ismain = 1
 116             elif m.group(1) == "A":
 117                 self.isneeded = 1
 118             else:
 119                 sys.exit('Internal error: Unknown match')
 120
 121             default = default[:len(default)-2]
 122
 123         if default and default[0] == '"':
 124             default = default[1:len(default)-1]
 125
 126         self.default = default
 127         # The value field will get the value passed to the argument when the tmeplate is used
 128         self.value = None
 129
 130
 131 # template is a class to store the whole definition of a genmf template
 132 # Members:
 133 # - name: name of the template
 134 # - args: an assosiative of the arguments of this template
 135 # - body: an array of strings with the body of the template
 136 # - mainarg: contains the arg with /M if it is present
 137 # - used: Is this template already used; used to check for recursive calling
 138 #   of a template
 139 # - linerefs: an array to indicate the genmf variables used in the body of the
 140 #   template. This is generated with the generate_linerefs method of this class
 141 # - templrefs: an array to indicate the templates used in the body of the template.
 142 #   This is generated with the generate_templrefs function of this class.
 143 class template:
 144     re_arginst = re.compile('%\(([a-zA-Z0-9][a-zA-Z0-9_]*)\)')
 145     hascommon = 0
 146
 147     # Generate a template
 148     # Input params:
 149     # - name: name of the template
 150     # - args: an assosiative array of the arguments defined for this template
 151     # - body: an array of the template with the bodylines of the template
 152     def __init__(self, name, args, body):
 153         self.name = name
 154         self.args = args
 155         self.body = body
 156         self.mainarg = None
 157         self.used = 0
 158         self.linerefs = None
 159         self.templrefs = None
 160
 161         for argname, argbody in args.items():
 162             if argbody.ismain:
 163                 if self.mainarg:
 164                     sys.exit('A template can have only one main (/M) argument')
 165                 self.mainarg = argbody
 166
 167     # Generate the references for the genmf variable used in this template
 168     # This function will return an assositive array of tuples with linerefs[lineno]
 169     # an array of tuples named argrefs with the tuple of the form (argbody, start, stop)
 170     # with argbody an object of class arg, start and stop the start and end of this variable
 171     # in the string of this line.
 172     def generate_linerefs(self):
 173         lineno = 0
 174         linerefs = {}
 175         while lineno < len(self.body):
 176             argrefs = []
 177             for m in template.re_arginst.finditer(self.body[lineno]):
 178                 if self.args.has_key(m.group(1)):
 179                     argbody = self.args[m.group(1)]
 180                     argrefs.append((argbody, m.start(), m.end()))
 181                     argbody.used = 1
 182
 183             if len(argrefs) > 0:
 184                 linerefs[lineno] = argrefs
 185
 186             lineno = lineno+1
 187         self.linerefs = linerefs
 188
 189         for argname, argbody in self.args.items():
 190             if not argbody.used:
 191                 sys.stderr.write("Warning: template '%s': unused argument '%s'\n" % (self.name, argname))
 192
 193
 194     # Write out the body of the template
 195     def write(self, outfile, name, line, templates):
 196         if self.used:
 197             raise GenmfException("Template '%s' called recursively" % name)
 198         self.used = 1
 199
 200         # Reading arguments of the template
 201         argno = 0
 202         while len(line) > 0:
 203             m = re_arg.match(line)
 204             if m and self.args.has_key(m.group(1)):
 205                 value = m.group(2)
 206                 if value == None:
 207                     #sys.stderr.write("Arg:"+m.group(1)+" Value: None Line:"+line+"\n")
 208                     self.args[m.group(1)].value = ''
 209                 else:
 210                     #sys.stderr.write("Arg:"+m.group(1)+" Value:"+m.group(2)+" Line:"+line+"\n")
 211                     if len(value)>0 and value[0] == '"':
 212                         value = value[1:len(value)-1]
 213                     self.args[m.group(1)].value = value
 214                 line = line[m.end():].lstrip()
 215             elif self.mainarg:
 216                 self.mainarg.value = line[:len(line)-1]
 217                 line = ''
 218             else:
 219                 raise GenmfException('Syntax error in arguments: '+line)
 220
 221         if self.linerefs == None:
 222             self.generate_linerefs()
 223             self.templrefs = generate_templrefs(self.body, templates)
 224
 225         for argname, argbody in self.args.items():
 226             if argbody.isneeded and argbody.value == None:
 227                 raise GenmfException('Arg "%s" not specified but should have been' % argname)
 228
 229         text = self.body[:]
 230
 231         for lineno, argrefs in self.linerefs.items():
 232             line = text[lineno]
 233
 234             pos=0
 235             lineout = ''
 236             for argref in argrefs:
 237                 if argref[1] > pos:
 238                     lineout = lineout + line[pos:argref[1]]
 239
 240                 if not argref[0].value == None:
 241                     lineout = lineout + argref[0].value
 242                 elif argref[0].default:
 243                     lineout = lineout + argref[0].default
 244
 245                 pos = argref[2]
 246
 247             if pos < len(line):
 248                 lineout = lineout + line[pos:]
 249
 250             text[lineno] = lineout
 251
 252         writelines(text, self.templrefs, templates, outfile)
 253         #outfile.write('\n')
 254
 255         for argname, argbody in self.args.items():
 256             argbody.value = None
 257         self.used = 0
 258
 259
 260
 261 # Read in the definition of the genmf templates from the given filename
 262 # Return an assosiative array of the templates present in this file.
 263 def read_templates(filename):
 264     try:
 265         infile = open(filename)
 266     except:
 267         print "Error reading template file: "+filename
 268
 269     re_name = re.compile('[a-zA-Z0-9][a-zA-Z0-9_]*(?=(?:\s|$))')
 270     re_openstring = re.compile('[^\s"]*"[^"]*$')
 271     re_define = re.compile('%define(?=\s)')
 272
 273     lines = infile.readlines()
 274     lineno = 0
 275     templates = {}
 276     while lineno < len(lines):
 277         line = lines[lineno]
 278         if re_define.match(line):
 279             while line[len(line)-2] == "\\" and lineno < len(lines):
 280                 lineno = lineno + 1
 281                 line = line[0:len(line)-2] + lines[lineno]
 282
 283             line = line[7:].strip()
 284
 285             m = re_name.match(line)
 286             if not m:
 287                 sys.exit("%s:%d:Error in syntax of template name" % (filename, lineno+1))
 288             tmplname = m.group(0)
 289             line = line[m.end():].lstrip()
 290
 291             args = {}
 292             while len(line) > 0:
 293                 m = re_arg.match(line)
 294                 if not m:
 295                     sys.exit("%s:%d:Error in syntax of argument %d Line: %s" % (filename, lineno+1, len(args)+1, line))
 296                 args[m.group(1)] = arg(m.group(2))
 297
 298                 line = line[m.end():].lstrip()
 299
 300             #print "Line: %d Template: %s" % (lineno+1, tmplname)
 301
 302             lineno = lineno+1
 303             line = lines[lineno]
 304             bodystart = lineno
 305             while lineno < len(lines) and line[0:4] <> "%end":
 306                 lineno = lineno+1
 307                 line = lines[lineno]
 308
 309             if lineno == len(lines):
 310                 sys.exit('%s:End of file reached in a template definition' % filename)
 311
 312             templates[tmplname] = template(tmplname, args, lines[bodystart:lineno])
 313
 314         lineno = lineno+1
 315
 316     return templates
 317
 318
 319 ################
 320 # Main program #
 321 ################
 322
 323 argv = []
 324 i = 0
 325 listfile = None
 326 while i < len(sys.argv):
 327     if sys.argv[i] == "--listfile":
 328         listfile = sys.argv[i+1]
 329         i = i + 2
 330     else:
 331         argv.append(sys.argv[i])
 332         i = i + 1
 333
 334 #sys.stderr.write("Reading templates\n")
 335 templates = read_templates(argv[1])
 336 #sys.stderr.write("Read %d templates\n" % len(templates))
 337
 338 if listfile == None:
 339     # Read on input file and write out one outputfile
 340     if len(sys.argv) == 2:
 341         lines = sys.stdin.readlines()
 342     else:
 343         infile = open(sys.argv[2], "r")
 344         lines = infile.readlines()
 345         infile.close()
 346
 347     if len(sys.argv) == 2:
 348         outfile = sys.stdout
 349         closeout = 0
 350     else:
 351         outfile = open(sys.argv[3], "w")
 352         closeout = 1
 353
 354     try:
 355         writelines(lines, generate_templrefs(lines, templates), templates, outfile)
 356     except GenmfException, ge:
 357         s = ge.s
 358         if len(sys.argv) == 4:
 359             s = sys.argv[3]+":"+s
 360             sys.exit(s+"\n")
 361
 362     # If %common was not present in the file write it out at the end of the file
 363     if not template.hascommon:
 364         outfile.write("\n")
 365         if templates.has_key("common"):
 366             templates["common"].write(outfile, "common", "", templates)
 367
 368     if closeout:
 369         outfile.close()
 370 else:
 371     # When a listfile is specified each line in this listfile is of the form
 372     # inputfile outputfile
 373     # and apply the instantiation of the templates to all these files listed there
 374     infile = open(listfile, "r")
 375     filelist = infile.readlines()
 376     infile.close()
 377
 378     for fileno in range(len(filelist)):
 379         files = filelist[fileno].split()
 380         if len(files) <> 2:
 381             sys.exit('%s:%d: Syntax error: %s' % (listfile, fileno+1, filelist[fileno]))
 382
 383         sys.stderr.write('Regenerating file %4d of %4d\r' % (fileno+1, len(filelist)))
 384         sys.stderr.flush()
 385
 386         infile = open(files[0], "r")
 387         lines = infile.readlines()
 388         infile.close()
 389
 390         outfile = open(files[1], "w")
 391         template.hascommon = 0
 392
 393         try:
 394             writelines(lines, generate_templrefs(lines, templates), templates, outfile)
 395         except GenmfException, ge:
 396             s = ge.s
 397             if len(sys.argv) == 4:
 398                 s = files[0]+":"+s
 399                 sys.exit(s+"\n")
 400
 401         if not template.hascommon:
 402             outfile.write("\n")
 403             if templates.has_key("common"):
 404                 templates["common"].write(outfile, "common", "", templates)
 405
 406         outfile.close()
 407
 408     sys.stderr.write('\n')