tools/genmf/genmf.py

   1 #! @PYTHON@
   2 # -*- coding: iso-8859-1 -*-
   3 # Copyright © 2003-2008, The AROS Development Team. All rights reserved.
   4
   5 import sys, re, os, errno
   6
   7 if not len(sys.argv) in [2, 4] :
   8     print "Usage:",sys.argv[0],"tmplfile [inputfile outputfile]"
   9     print "Usage:",sys.argv[0],"tmplfile --listfile filename"
  10
  11 # A regular expression for the start of a template instantiation (ex. %build_module)
  12 re_tmplinst = re.compile('%([a-zA-Z0-9][a-zA-Z0-9_]*)(?=(?:\s|$))')
  13 # A regular expression for the argument specification during template instantiation
  14 # (ex. cflags=$(CFLAGS) or uselibs="amiga arosc")
  15 re_arg = re.compile('([a-zA-Z0-9][a-zA-Z0-9_]*)=([^\s"]+|".*?")?')
  16
  17 ##################################
  18 # Class and function definitions #
  19 ##################################
  20
  21 # Exception used throughout this program
  22 class GenmfException:
  23     def __init__(self, s):
  24         self.s = s
  25     def __str__(self):
  26         return self.s
  27
  28
  29 # Scan the given lines for template instantiations
  30 # Input params:
  31 # - lines: an array of strings
  32 # - templates: an assosiative array of objects of class template
  33 #
  34 # Returns:
  35 # - templrefs: an array of tuples with two elements. First element is the line number
  36 #   in the lines array, second is the result of the search for that line with
  37 #   re_tmplinst. Only templates which name are present in the templates argument
  38 #   will be added to this array.
  39 def generate_templrefs(lines, templates):
  40     templrefs = []
  41
  42     for lineno in range(len(lines)):
  43         line = lines[lineno]
  44         if len(line) == 0 or line[0] == "#":
  45             continue
  46
  47         m = re_tmplinst.search(line)
  48         if m and templates.has_key(m.group(1)) and not (m.start() > 0 and line[m.start()-1] == "#"):
  49             templrefs.append((lineno, m))
  50
  51     return templrefs
  52
  53
  54 # Write out the lines to a file with instantiating the templates present in the file
  55 # Input params:
  56 # - lines: The lines to write out
  57 # - templrefs: the instantiated templates present in these lines
  58 # - templates: the template definitions
  59 # - outfile: the file to write to
  60 #
  61 # This function does not return anything but raises a GenmfException when there are
  62 # problems
  63 def writelines(lines, templrefs, templates, outfile):
  64     start = 0
  65     for lineno, m in templrefs:
  66         if start<lineno:
  67             outfile.writelines(lines[start:lineno])
  68
  69         start = lineno + 1
  70         line = lines[lineno]
  71         while line[len(line)-2] == "\\" and start < len(lines):
  72             line = line[0:len(line)-2] + lines[start]
  73             start = start + 1
  74
  75         if m.group(1) == "common":
  76             template.hascommon = 1
  77
  78         try:
  79             templates[m.group(1)].write(outfile, m.group(1), line[m.end():].lstrip(), templates)
  80         except GenmfException, ge:
  81             raise GenmfException(("In instantiation of %s, line %d\n" % (m.group(1), lineno+1))+ge.s)
  82
  83     if start < len(lines):
  84         outfile.writelines(lines[start:len(lines)])
  85
  86
  87 # arg is a class that stores the specification of an argument in the template header.
  88 # The name of the arg is not stored in this class but this class is supposed to be
  89 # stored in an assosiative array with the names of the arg as indices. E.g
  90 # arg['cflags'] gives back an object of this class.
  91 # It has the following members:
  92 # - ismulti: boolean to indicate if it has the /M definition
  93 # - isneeded: boolean to indicate if it has the /A definition
  94 # - used: boolean to indicate if a value is used in the body of a template
  95 # - default: default value when the argument is not given during a template
  96 #   instantiation
  97 # - value: The value this argument has during a template instantiation is stored
  98 #   here
  99 class arg:
 100     # Specification can end with /A or /M
 101     re_mode = re.compile('/(A|M)')
 102
 103     # You create this object with giving it the default value
 104     def __init__(self, default=None):
 105         self.ismulti = 0
 106         self.isneeded = 0
 107         self.used = 0
 108
 109         while default and len(default)>1:
 110             m = arg.re_mode.match(default[len(default)-2:])
 111             if not m:
 112                 break
 113             if m.group(1) == "M":
 114                 self.ismulti = 1
 115             elif m.group(1) == "A":
 116                 self.isneeded = 1
 117             else:
 118                 sys.exit('Internal error: Unknown match')
 119
 120             default = default[:len(default)-2]
 121
 122         if default and default[0] == '"':
 123             default = default[1:len(default)-1]
 124
 125         self.default = default
 126         # The value field will get the value passed to the argument when the template is used
 127         self.value = None
 128
 129
 130 # template is a class to store the whole definition of a genmf template
 131 # Members:
 132 # - name: name of the template
 133 # - args: an associative array of the arguments of this template
 134 # - body: an array of strings with the body of the template
 135 # - multiarg: contains the arg with /M if it is present
 136 # - used: Is this template already used; used to check for recursive calling
 137 #   of a template
 138 # - linerefs: an array to indicate the genmf variables used in the body of the
 139 #   template. This is generated with the generate_linerefs method of this class
 140 # - templrefs: an array to indicate the templates used in the body of the template.
 141 #   This is generated with the generate_templrefs function of this class.
 142 class template:
 143     re_arginst = re.compile('([a-zA-Z0-9-_]*)%\(([a-zA-Z0-9][a-zA-Z0-9_]*)\)([a-zA-Z0-9-_]*)')
 144     hascommon = 0
 145
 146     # Generate a template
 147     # Input params:
 148     # - name: name of the template
 149     # - args: an assosiative array of the arguments defined for this template
 150     # - body: an array of the template with the bodylines of the template
 151     def __init__(self, name, args, body):
 152         self.name = name
 153         self.args = args
 154         self.body = body
 155         self.multiarg = None
 156         self.used = 0
 157         self.linerefs = None
 158         self.templrefs = None
 159
 160         for argname, argbody in args.items():
 161             if argbody.ismulti:
 162                 if self.multiarg:
 163                     sys.exit('A template can have only one main (/M) argument')
 164                 self.multiarg = argbody
 165
 166     # Generate the references for the genmf variable used in this template
 167     # This function will return an assositive array of tuples with linerefs[lineno]
 168     # an array of tuples named argrefs with the tuple of the form (argbody, start, stop, prefix, suffix)
 169     # with argbody an object of class arg, start and stop the start and end of this variable
 170     # in the string of this line.
 171     def generate_linerefs(self):
 172         lineno = 0
 173         linerefs = {}
 174         while lineno < len(self.body):
 175             argrefs = []
 176             for m in template.re_arginst.finditer(self.body[lineno]):
 177                 if self.args.has_key(m.group(2)):
 178                     argbody = self.args[m.group(2)]
 179                     argrefs.append((argbody, m.start(), m.end(),m.group(1),m.group(3)))
 180                     argbody.used = 1
 181
 182             if len(argrefs) > 0:
 183                 linerefs[lineno] = argrefs
 184
 185             lineno = lineno+1
 186         self.linerefs = linerefs
 187
 188         for argname, argbody in self.args.items():
 189             if not argbody.used:
 190                 sys.stderr.write("Warning: template '%s': unused argument '%s'\n" % (self.name, argname))
 191
 192
 193     # Write out the body of the template
 194     def write(self, outfile, name, line, templates):
 195         if self.used:
 196             raise GenmfException("Template '%s' called recursively" % name)
 197         self.used = 1
 198
 199         # Reading arguments of the template
 200         argno = 0
 201         while len(line) > 0:
 202             m = re_arg.match(line)
 203             if m and self.args.has_key(m.group(1)):
 204                 value = m.group(2)
 205                 if value == None:
 206                     #sys.stderr.write("Arg:"+m.group(1)+" Value: None Line:"+line+"\n")
 207                     self.args[m.group(1)].value = ''
 208                 else:
 209                     #sys.stderr.write("Arg:"+m.group(1)+" Value:"+m.group(2)+" Line:"+line+"\n")
 210                     if len(value)>0 and value[0] == '"':
 211                         value = value[1:len(value)-1]
 212                     self.args[m.group(1)].value = value
 213                 line = line[m.end():].lstrip()
 214             elif self.multiarg:
 215                 self.multiarg.value = line[:len(line)-1]
 216                 line = ''
 217             else:
 218                 raise GenmfException('Syntax error in arguments: '+line)
 219
 220         if self.linerefs == None:
 221             self.generate_linerefs()
 222             self.templrefs = generate_templrefs(self.body, templates)
 223
 224         for argname, argbody in self.args.items():
 225             if argbody.isneeded and argbody.value == None:
 226                 raise GenmfException('Arg "%s" not specified but should have been' % argname)
 227
 228         text = self.body[:]
 229
 230         for lineno, argrefs in self.linerefs.items():
 231             line = text[lineno]
 232
 233             pos=0
 234             lineout = ''
 235             for argref in argrefs:
 236                 if argref[1] > pos:
 237                     lineout = lineout + line[pos:argref[1]]
 238
 239                 linevals=[]
 240                 if not argref[0].value == None:
 241                     for val in argref[0].value.split():
 242                         linevals.append(argref[3] + val + argref[4])
 243                 elif argref[0].default:
 244                     for val in argref[0].default.split():
 245                         linevals.append(argref[3] + val + argref[4])
 246
 247                 if len(linevals) == 0:
 248                     lineout += argref[3] + argref[4]
 249                 else:
 250                     lineout += " ".join(linevals)
 251
 252                 pos = argref[2]
 253
 254             if pos < len(line):
 255                 lineout = lineout + line[pos:]
 256
 257             text[lineno] = lineout
 258
 259         writelines(text, self.templrefs, templates, outfile)
 260         #outfile.write('\n')
 261
 262         for argname, argbody in self.args.items():
 263             argbody.value = None
 264         self.used = 0
 265
 266
 267
 268 # Read in the definition of the genmf templates from the given filename
 269 # Return an assosiative array of the templates present in this file.
 270 def read_templates(filename):
 271     try:
 272         infile = open(filename)
 273     except:
 274         print "Error reading template file: "+filename
 275
 276     re_name = re.compile('[a-zA-Z0-9][a-zA-Z0-9_]*(?=(?:\s|$))')
 277     re_openstring = re.compile('[^\s"]*"[^"]*$')
 278     re_define = re.compile('%define(?=\s)')
 279
 280     lines = infile.readlines()
 281     lineno = 0
 282     templates = {}
 283     while lineno < len(lines):
 284         line = lines[lineno]
 285         if re_define.match(line):
 286             while line[len(line)-2] == "\\" and lineno < len(lines):
 287                 lineno = lineno + 1
 288                 line = line[0:len(line)-2] + lines[lineno]
 289
 290             line = line[7:].strip()
 291
 292             m = re_name.match(line)
 293             if not m:
 294                 sys.exit("%s:%d:Error in syntax of template name" % (filename, lineno+1))
 295             tmplname = m.group(0)
 296             line = line[m.end():].lstrip()
 297
 298             args = {}
 299             while len(line) > 0:
 300                 m = re_arg.match(line)
 301                 if not m:
 302                     sys.exit("%s:%d:Error in syntax of argument %d Line: %s" % (filename, lineno+1, len(args)+1, line))
 303                 args[m.group(1)] = arg(m.group(2))
 304
 305                 line = line[m.end():].lstrip()
 306
 307             #print "Line: %d Template: %s" % (lineno+1, tmplname)
 308
 309             lineno = lineno+1
 310             line = lines[lineno]
 311             bodystart = lineno
 312             while lineno < len(lines) and line[0:4] <> "%end":
 313                 lineno = lineno+1
 314                 line = lines[lineno]
 315
 316             if lineno == len(lines):
 317                 sys.exit('%s:End of file reached in a template definition' % filename)
 318
 319             templates[tmplname] = template(tmplname, args, lines[bodystart:lineno])
 320
 321         lineno = lineno+1
 322
 323     return templates
 324
 325
 326 ################
 327 # Main program #
 328 ################
 329
 330 argv = []
 331 i = 0
 332 listfile = None
 333 while i < len(sys.argv):
 334     if sys.argv[i] == "--listfile":
 335         listfile = sys.argv[i+1]
 336         i = i + 2
 337     else:
 338         argv.append(sys.argv[i])
 339         i = i + 1
 340
 341 #sys.stderr.write("Reading templates\n")
 342 templates = read_templates(argv[1])
 343 #sys.stderr.write("Read %d templates\n" % len(templates))
 344
 345 if listfile == None:
 346     # Read one input file and write out one outputfile
 347     if len(sys.argv) == 2:
 348         lines = sys.stdin.readlines()
 349     else:
 350         infile = open(sys.argv[2], "r")
 351         lines = infile.readlines()
 352         infile.close()
 353
 354     if len(sys.argv) == 2:
 355         outfile = sys.stdout
 356         closeout = 0
 357     else:
 358         outfile = open(sys.argv[3], "w")
 359         closeout = 1
 360
 361     try:
 362         writelines(lines, generate_templrefs(lines, templates), templates, outfile)
 363     except GenmfException, ge:
 364         s = ge.s
 365         if len(sys.argv) == 4:
 366             s = sys.argv[3]+":"+s
 367             sys.exit(s+"\n")
 368
 369     # If %common was not present in the file write it out at the end of the file
 370     if not template.hascommon:
 371         outfile.write("\n")
 372         if templates.has_key("common"):
 373             templates["common"].write(outfile, "common", "", templates)
 374
 375     if closeout:
 376         outfile.close()
 377 else:
 378     # When a listfile is specified each line in this listfile is of the form
 379     # inputfile outputfile
 380     # Apply the instantiation of the templates to all these files listed there
 381     infile = open(listfile, "r")
 382     filelist = infile.readlines()
 383     infile.close()
 384
 385     for fileno in range(len(filelist)):
 386         files = filelist[fileno].split()
 387         if len(files) <> 2:
 388             sys.exit('%s:%d: Syntax error: %s' % (listfile, fileno+1, filelist[fileno]))
 389
 390         sys.stderr.write('Regenerating file %4d of %4d\r' % (fileno+1, len(filelist)))
 391         sys.stderr.flush()
 392
 393         infile = open(files[0], "r")
 394         lines = infile.readlines()
 395         infile.close()
 396
 397         try:
 398             # os.makedirs will also create all the parent directories
 399             os.makedirs(os.path.dirname(files[1]))
 400         except OSError, err:
 401             # Do nothing ..
 402             s = err.errno
 403
 404         outfile = open(files[1], "w")
 405         template.hascommon = 0
 406
 407         try:
 408             writelines(lines, generate_templrefs(lines, templates), templates, outfile)
 409         except GenmfException, ge:
 410             s = ge.s
 411             if len(sys.argv) == 4:
 412                 s = files[0]+":"+s
 413                 sys.exit(s+"\n")
 414
 415         if not template.hascommon:
 416             outfile.write("\n")
 417             if templates.has_key("common"):
 418                 templates["common"].write(outfile, "common", "", templates)
 419
 420         outfile.close()
 421
 422     sys.stderr.write('\n')