tools/genmf/genmf.py

   1 #! @PYTHON@
   2 # -*- coding: iso-8859-1 -*-
   3 # Copyright © 2003-2008, The AROS Development Team. All rights reserved.
   4
   5 import sys, re, os, errno
   6
   7 if not len(sys.argv) in [2, 4] :
   8     print "Usage:",sys.argv[0],"tmplfile [inputfile outputfile]"
   9     print "Usage:",sys.argv[0],"tmplfile --listfile filename"
  10
  11 # A regular expression for the start of a template instantiation (ex. %build_module)
  12 re_tmplinst = re.compile('%([a-zA-Z0-9][a-zA-Z0-9_]*)(?=(?:\s|$))')
  13 # A regular expression for the argument specification during template instantiation
  14 # (ex. cflags=$(CFLAGS) or uselibs="amiga arosc")
  15 re_arg = re.compile('([a-zA-Z0-9][a-zA-Z0-9_]*)=([^\s"]+|".*?")?')
  16
  17 ##################################
  18 # Class and function definitions #
  19 ##################################
  20
  21 # Exception used throughout this program
  22 class GenmfException:
  23     def __init__(self, s):
  24         self.s = s
  25     def __str__(self):
  26         return self.s
  27
  28
  29 # Scan the given lines for template instantiations
  30 # Input params:
  31 # - lines: an array of strings
  32 # - templates: an assosiative array of objects of class template
  33 #
  34 # Returns:
  35 # - templrefs: an array of tuples with two elems. First element is the lineno
  36 #   in the lines array, second is the results of the search for that line with
  37 #   re_tmplinst. Only templates which name are present in the templates argument
  38 #   will be added to this array.
  39 def generate_templrefs(lines, templates):
  40     templrefs = []
  41
  42     for lineno in range(len(lines)):
  43         line = lines[lineno]
  44         if len(line) == 0 or line[0] == "#":
  45             continue
  46
  47         m = re_tmplinst.search(line)
  48         if m and templates.has_key(m.group(1)) and not (m.start() > 0 and line[m.start()-1] == "#"):
  49             templrefs.append((lineno, m))
  50
  51     return templrefs
  52
  53
  54 # Write out the lines to a file with instantiating the templates present in the file
  55 # Input params:
  56 # - lines: The lines to write out
  57 # - templrefs: the instantiated templates present in these lines
  58 # - templates: the template definitions
  59 # - outfile: the file to write to
  60 #
  61 # This function does not return anything but raises a GenmfException when there are
  62 # problems
  63 def writelines(lines, templrefs, templates, outfile):
  64     start = 0
  65     for lineno, m in templrefs:
  66         if start<lineno:
  67             outfile.writelines(lines[start:lineno])
  68
  69         start = lineno + 1
  70         line = lines[lineno]
  71         while line[len(line)-2] == "\\" and start < len(lines):
  72             line = line[0:len(line)-2] + lines[start]
  73             start = start + 1
  74
  75         if m.group(1) == "common":
  76             template.hascommon = 1
  77
  78         try:
  79             templates[m.group(1)].write(outfile, m.group(1), line[m.end():].lstrip(), templates)
  80         except GenmfException, ge:
  81             raise GenmfException(("In instantiation of %s, line %d\n" % (m.group(1), lineno+1))+ge.s)
  82
  83     if start < len(lines):
  84         outfile.writelines(lines[start:len(lines)])
  85
  86
  87 # arg is q class that stores the specification of an argument in the template header
  88 # The of the arg is not stored in this class but this class is supposed to be
  89 # stored in an assosiative array with the names of the arg as indices. E.g
  90 # arg['cflags'] gives back an object of this class.
  91 # It has the following members:
  92 # - ismain: boolean to indicate if it has the /M definition
  93 # - isneeded: boolean to indicate if it has the /A definition
  94 # - used: boolean to indicate if a value is used in the body of a template
  95 # - default: default value when the argument is not given during a template
  96 #   instantiation
  97 # - value: The value this argument has during a template instantiation is stored
  98 #   here
  99 class arg:
 100     # Specification can end with /A or /M
 101     re_mode = re.compile('/(A|M)')
 102
 103     # You create this object with giving it the default value
 104     def __init__(self, default=None):
 105         self.ismain = 0
 106         self.isneeded = 0
 107         self.used = 0
 108
 109         while default and len(default)>1:
 110             m = arg.re_mode.match(default[len(default)-2:])
 111             if not m:
 112                 break
 113             if m.group(1) == "M":
 114                 self.ismain = 1
 115             elif m.group(1) == "A":
 116                 self.isneeded = 1
 117             else:
 118                 sys.exit('Internal error: Unknown match')
 119
 120             default = default[:len(default)-2]
 121
 122         if default and default[0] == '"':
 123             default = default[1:len(default)-1]
 124
 125         self.default = default
 126         # The value field will get the value passed to the argument when the tmeplate is used
 127         self.value = None
 128
 129
 130 # template is a class to store the whole definition of a genmf template
 131 # Members:
 132 # - name: name of the template
 133 # - args: an assosiative of the arguments of this template
 134 # - body: an array of strings with the body of the template
 135 # - mainarg: contains the arg with /M if it is present
 136 # - used: Is this template already used; used to check for recursive calling
 137 #   of a template
 138 # - linerefs: an array to indicate the genmf variables used in the body of the
 139 #   template. This is generated with the generate_linerefs method of this class
 140 # - templrefs: an array to indicate the templates used in the body of the template.
 141 #   This is generated with the generate_templrefs function of this class.
 142 class template:
 143     re_arginst = re.compile('%\(([a-zA-Z0-9][a-zA-Z0-9_]*)\)')
 144     hascommon = 0
 145
 146     # Generate a template
 147     # Input params:
 148     # - name: name of the template
 149     # - args: an assosiative array of the arguments defined for this template
 150     # - body: an array of the template with the bodylines of the template
 151     def __init__(self, name, args, body):
 152         self.name = name
 153         self.args = args
 154         self.body = body
 155         self.mainarg = None
 156         self.used = 0
 157         self.linerefs = None
 158         self.templrefs = None
 159
 160         for argname, argbody in args.items():
 161             if argbody.ismain:
 162                 if self.mainarg:
 163                     sys.exit('A template can have only one main (/M) argument')
 164                 self.mainarg = argbody
 165
 166     # Generate the references for the genmf variable used in this template
 167     # This function will return an assositive array of tuples with linerefs[lineno]
 168     # an array of tuples named argrefs with the tuple of the form (argbody, start, stop)
 169     # with argbody an object of class arg, start and stop the start and end of this variable
 170     # in the string of this line.
 171     def generate_linerefs(self):
 172         lineno = 0
 173         linerefs = {}
 174         while lineno < len(self.body):
 175             argrefs = []
 176             for m in template.re_arginst.finditer(self.body[lineno]):
 177                 if self.args.has_key(m.group(1)):
 178                     argbody = self.args[m.group(1)]
 179                     argrefs.append((argbody, m.start(), m.end()))
 180                     argbody.used = 1
 181
 182             if len(argrefs) > 0:
 183                 linerefs[lineno] = argrefs
 184
 185             lineno = lineno+1
 186         self.linerefs = linerefs
 187
 188         for argname, argbody in self.args.items():
 189             if not argbody.used:
 190                 sys.stderr.write("Warning: template '%s': unused argument '%s'\n" % (self.name, argname))
 191
 192
 193     # Write out the body of the template
 194     def write(self, outfile, name, line, templates):
 195         if self.used:
 196             raise GenmfException("Template '%s' called recursively" % name)
 197         self.used = 1
 198
 199         # Reading arguments of the template
 200         argno = 0
 201         while len(line) > 0:
 202             m = re_arg.match(line)
 203             if m and self.args.has_key(m.group(1)):
 204                 value = m.group(2)
 205                 if value == None:
 206                     #sys.stderr.write("Arg:"+m.group(1)+" Value: None Line:"+line+"\n")
 207                     self.args[m.group(1)].value = ''
 208                 else:
 209                     #sys.stderr.write("Arg:"+m.group(1)+" Value:"+m.group(2)+" Line:"+line+"\n")
 210                     if len(value)>0 and value[0] == '"':
 211                         value = value[1:len(value)-1]
 212                     self.args[m.group(1)].value = value
 213                 line = line[m.end():].lstrip()
 214             elif self.mainarg:
 215                 self.mainarg.value = line[:len(line)-1]
 216                 line = ''
 217             else:
 218                 raise GenmfException('Syntax error in arguments: '+line)
 219
 220         if self.linerefs == None:
 221             self.generate_linerefs()
 222             self.templrefs = generate_templrefs(self.body, templates)
 223
 224         for argname, argbody in self.args.items():
 225             if argbody.isneeded and argbody.value == None:
 226                 raise GenmfException('Arg "%s" not specified but should have been' % argname)
 227
 228         text = self.body[:]
 229
 230         for lineno, argrefs in self.linerefs.items():
 231             line = text[lineno]
 232
 233             pos=0
 234             lineout = ''
 235             for argref in argrefs:
 236                 if argref[1] > pos:
 237                     lineout = lineout + line[pos:argref[1]]
 238
 239                 if not argref[0].value == None:
 240                     lineout = lineout + argref[0].value
 241                 elif argref[0].default:
 242                     lineout = lineout + argref[0].default
 243
 244                 pos = argref[2]
 245
 246             if pos < len(line):
 247                 lineout = lineout + line[pos:]
 248
 249             text[lineno] = lineout
 250
 251         writelines(text, self.templrefs, templates, outfile)
 252         #outfile.write('\n')
 253
 254         for argname, argbody in self.args.items():
 255             argbody.value = None
 256         self.used = 0
 257
 258
 259
 260 # Read in the definition of the genmf templates from the given filename
 261 # Return an assosiative array of the templates present in this file.
 262 def read_templates(filename):
 263     try:
 264         infile = open(filename)
 265     except:
 266         print "Error reading template file: "+filename
 267
 268     re_name = re.compile('[a-zA-Z0-9][a-zA-Z0-9_]*(?=(?:\s|$))')
 269     re_openstring = re.compile('[^\s"]*"[^"]*$')
 270     re_define = re.compile('%define(?=\s)')
 271
 272     lines = infile.readlines()
 273     lineno = 0
 274     templates = {}
 275     while lineno < len(lines):
 276         line = lines[lineno]
 277         if re_define.match(line):
 278             while line[len(line)-2] == "\\" and lineno < len(lines):
 279                 lineno = lineno + 1
 280                 line = line[0:len(line)-2] + lines[lineno]
 281
 282             line = line[7:].strip()
 283
 284             m = re_name.match(line)
 285             if not m:
 286                 sys.exit("%s:%d:Error in syntax of template name" % (filename, lineno+1))
 287             tmplname = m.group(0)
 288             line = line[m.end():].lstrip()
 289
 290             args = {}
 291             while len(line) > 0:
 292                 m = re_arg.match(line)
 293                 if not m:
 294                     sys.exit("%s:%d:Error in syntax of argument %d Line: %s" % (filename, lineno+1, len(args)+1, line))
 295                 args[m.group(1)] = arg(m.group(2))
 296
 297                 line = line[m.end():].lstrip()
 298
 299             #print "Line: %d Template: %s" % (lineno+1, tmplname)
 300
 301             lineno = lineno+1
 302             line = lines[lineno]
 303             bodystart = lineno
 304             while lineno < len(lines) and line[0:4] <> "%end":
 305                 lineno = lineno+1
 306                 line = lines[lineno]
 307
 308             if lineno == len(lines):
 309                 sys.exit('%s:End of file reached in a template definition' % filename)
 310
 311             templates[tmplname] = template(tmplname, args, lines[bodystart:lineno])
 312
 313         lineno = lineno+1
 314
 315     return templates
 316
 317
 318 ################
 319 # Main program #
 320 ################
 321
 322 argv = []
 323 i = 0
 324 listfile = None
 325 while i < len(sys.argv):
 326     if sys.argv[i] == "--listfile":
 327         listfile = sys.argv[i+1]
 328         i = i + 2
 329     else:
 330         argv.append(sys.argv[i])
 331         i = i + 1
 332
 333 #sys.stderr.write("Reading templates\n")
 334 templates = read_templates(argv[1])
 335 #sys.stderr.write("Read %d templates\n" % len(templates))
 336
 337 if listfile == None:
 338     # Read one input file and write out one outputfile
 339     if len(sys.argv) == 2:
 340         lines = sys.stdin.readlines()
 341     else:
 342         infile = open(sys.argv[2], "r")
 343         lines = infile.readlines()
 344         infile.close()
 345
 346     if len(sys.argv) == 2:
 347         outfile = sys.stdout
 348         closeout = 0
 349     else:
 350         outfile = open(sys.argv[3], "w")
 351         closeout = 1
 352
 353     try:
 354         writelines(lines, generate_templrefs(lines, templates), templates, outfile)
 355     except GenmfException, ge:
 356         s = ge.s
 357         if len(sys.argv) == 4:
 358             s = sys.argv[3]+":"+s
 359             sys.exit(s+"\n")
 360
 361     # If %common was not present in the file write it out at the end of the file
 362     if not template.hascommon:
 363         outfile.write("\n")
 364         if templates.has_key("common"):
 365             templates["common"].write(outfile, "common", "", templates)
 366
 367     if closeout:
 368         outfile.close()
 369 else:
 370     # When a listfile is specified each line in this listfile is of the form
 371     # inputfile outputfile
 372     # and apply the instantiation of the templates to all these files listed there
 373     infile = open(listfile, "r")
 374     filelist = infile.readlines()
 375     infile.close()
 376
 377     for fileno in range(len(filelist)):
 378         files = filelist[fileno].split()
 379         if len(files) <> 2:
 380             sys.exit('%s:%d: Syntax error: %s' % (listfile, fileno+1, filelist[fileno]))
 381
 382         sys.stderr.write('Regenerating file %4d of %4d\r' % (fileno+1, len(filelist)))
 383         sys.stderr.flush()
 384
 385         infile = open(files[0], "r")
 386         lines = infile.readlines()
 387         infile.close()
 388
 389         try:
 390             # os.makedirs will also create all the parent directories
 391             os.makedirs(os.path.dirname(files[1]))
 392         except OSError, err:
 393             # Do nothing ..
 394             s = err.errno
 395
 396         outfile = open(files[1], "w")
 397         template.hascommon = 0
 398
 399         try:
 400             writelines(lines, generate_templrefs(lines, templates), templates, outfile)
 401         except GenmfException, ge:
 402             s = ge.s
 403             if len(sys.argv) == 4:
 404                 s = files[0]+":"+s
 405                 sys.exit(s+"\n")
 406
 407         if not template.hascommon:
 408             outfile.write("\n")
 409             if templates.has_key("common"):
 410                 templates["common"].write(outfile, "common", "", templates)
 411
 412         outfile.close()
 413
 414     sys.stderr.write('\n')