ConfigFile.py

   1 #
   2 # Stuff for dealing with configuration files.
   3 #
   4 #
   5 # This code is part of the LWN git data miner.
   6 #
   7 # Copyright 2007-11 Eklektix, Inc.
   8 # Copyright 2007-11 Jonathan Corbet <corbet@lwn.net>
   9 #
  10 # This file may be distributed under the terms of the GNU General
  11 # Public License, version 2.
  12 #
  13 import sys, re, datetime, os.path
  14 import database, sre_constants
  15
  16 #
  17 # Read a line and strip out junk.
  18 #
  19 def ReadConfigLine(file):
  20     line = file.readline()
  21     if not line:
  22         return None
  23     line = line.split('#')[0] # Get rid of any comments
  24     line = line.strip() # and extra white space
  25     if len(line) == 0: # we got rid of everything
  26         return ReadConfigLine(file)
  27     return line
  28
  29 #
  30 # Give up and die.
  31 #
  32 def croak(message):
  33     sys.stderr.write(message + '\n')
  34     sys.exit(1)
  35
  36 #
  37 # Read a list of email aliases.
  38 #
  39 def ReadEmailAliases(name):
  40     try:
  41         file = open(name, 'r')
  42     except IOError:
  43         croak('Unable to open email alias file %s' % (name))
  44     line = ReadConfigLine(file)
  45     while line:
  46         m = re.match(r'^("[^"]+"|\S+)\s+(.+)$', line)
  47         if not m or len(m.groups()) != 2:
  48             croak('Funky email alias line "%s"' % (line))
  49         if m and m.group(2).find('@') <= 0:
  50             croak('Non-addresses in email alias "%s"' % (line))
  51         database.AddEmailAlias(m.group(1).replace('"', ''), m.group(2))
  52         line = ReadConfigLine(file)
  53     file.close()
  54
  55 def ReadRegexpEmailAliases(name):
  56     try:
  57         file = open(name, 'r')
  58     except IOError:
  59         croak('Unable to open regex email alias file %s' % (name))
  60     line = ReadConfigLine(file)
  61     while line:
  62         m = re.match(r'^("[^"]+"|\S+)\s+(.+)$', line)
  63         if not m or len(m.groups()) != 2:
  64             croak('Funky email alias line "%s"' % (line))
  65         if m and m.group(2).find('@') <= 0:
  66             croak('Non-addresses in email alias "%s"' % (line))
  67         try:
  68             regex = re.compile(m.group(1).replace('"', ''))
  69         except sre_constants.error:
  70             croak('Screwy regex: %s' % (m.group(1)))
  71         database.AddRXEmailAlias(regex, m.group(2))
  72         line = ReadConfigLine(file)
  73     file.close()
  74
  75
  76 #
  77 # The Email/Employer map
  78 #
  79 EMMpat = re.compile(r'^([^\s]+)\s+([^<]+)\s*(<\s*(\d+-\d+-\d+)\s*)?$')
  80
  81 def ReadEmailEmployers(name):
  82     try:
  83         file = open(name, 'r')
  84     except IOError:
  85         croak('Unable to open email/employer file %s' % (name))
  86     line = ReadConfigLine(file)
  87     while line:
  88         m = EMMpat.match(line)
  89         if not m:
  90             croak('Funky email/employer line "%s"' % (line))
  91         email = m.group(1)
  92         company = m.group(2).strip()
  93         enddate = ParseDate(m.group(4))
  94         database.AddEmailEmployerMapping(email, company, enddate)
  95         line = ReadConfigLine(file)
  96     file.close()
  97
  98 def ParseDate(cdate):
  99     if not cdate:
 100         return None
 101     sdate = cdate.split('-')
 102     return datetime.date(int(sdate[0]), int(sdate[1]), int(sdate[2]))
 103
 104
 105 def ReadGroupMap(fname, employer):
 106     try:
 107         file = open(fname, 'r')
 108     except IOError:
 109         croak('Unable to open group map file %s' % (fname))
 110     line = ReadConfigLine(file)
 111     while line:
 112         database.AddEmailEmployerMapping(line, employer)
 113         line = ReadConfigLine(file)
 114     file.close()
 115
 116 #
 117 # Read in a virtual employer description.
 118 #
 119 def ReadVirtual(file, name):
 120     ve = database.VirtualEmployer(name)
 121     line = ReadConfigLine(file)
 122     while line:
 123         sl = line.split(None, 1)
 124         first = sl[0]
 125         if first == 'end':
 126             ve.store()
 127             return
 128         #
 129         # Zap the "%" syntactic sugar if it's there
 130         #
 131         if first[-1] == '%':
 132             first = first[:-1]
 133         try:
 134             percent = int(first)
 135         except ValueError:
 136             croak('Bad split value "%s" for virtual empl %s' % (first, name))
 137         if not (0 < percent <= 100):
 138             croak('Bad split value "%s" for virtual empl %s' % (first, name))
 139         ve.addsplit(' '.join(sl[1:]), percent/100.0)
 140         line = ReadConfigLine(file)
 141     #
 142     # We should never get here
 143     #
 144     croak('Missing "end" line for virtual employer %s' % (name))
 145
 146 #
 147 # Read file type patterns for more fine graned reports
 148 #
 149 def ReadFileType(filename):
 150     try:
 151         file = open(filename, 'r')
 152     except IOError:
 153         croak('Unable to open file type mapping file %s' % (filename))
 154     patterns = {}
 155     order = []
 156     regex_order = re.compile(r'^order\s+(.*)$')
 157     regex_file_type = re.compile(r'^filetype\s+(\S+)\s+(.+)$')
 158     line = ReadConfigLine(file)
 159     while line:
 160         o = regex_order.match(line)
 161         if o:
 162             # Consider only the first definition in the config file
 163             elements = o.group(1).replace(' ', '')
 164             order = order or elements.split(',')
 165             line = ReadConfigLine(file)
 166             continue
 167
 168         m = regex_file_type.match(line)
 169         if not m or len(m.groups()) != 2:
 170             croak('Funky file type line "%s"' % (line))
 171         if m.group(1) not in patterns:
 172             patterns[m.group(1)] = []
 173         if m.group(1) not in order:
 174             print('%s not found, appended to the last order' % m.group(1))
 175             order.append(m.group(1))
 176
 177         patterns[m.group(1)].append(re.compile(m.group(2), re.IGNORECASE))
 178
 179         line = ReadConfigLine(file)
 180     file.close()
 181     return patterns, order
 182
 183 #
 184 # Read an overall config file.
 185 #
 186
 187 def ConfigFile(name, confdir):
 188     try:
 189         file = open(name, 'r')
 190     except IOError:
 191         try:
 192             file = open(os.path.join(confdir, name), 'r')
 193         except IOError:
 194             croak('Unable to open config file %s' % (name))
 195     line = ReadConfigLine(file)
 196     while line:
 197         sline = line.split(None, 2)
 198         if len(sline) < 2:
 199             croak('Funky config line: "%s"' % (line))
 200         if sline[0] == 'EmailAliases':
 201             ReadEmailAliases(os.path.join(confdir, sline[1]))
 202         elif sline[0] == 'RXEmailAliases':
 203             ReadRegexpEmailAliases(os.path.join(confdir, sline[1]))
 204         elif sline[0] == 'EmailMap':
 205             ReadEmailEmployers(os.path.join(confdir, sline[1]))
 206         elif sline[0] == 'GroupMap':
 207             if len(sline) != 3:
 208                 croak('Funky group map line "%s"' % (line))
 209             ReadGroupMap(os.path.join(confdir, sline[1]), sline[2])
 210         elif sline[0] == 'VirtualEmployer':
 211             ReadVirtual(file, ' '.join(sline[1:]))
 212         elif sline[0] == 'FileTypeMap':
 213             patterns, order = ReadFileType(os.path.join(confdir, sline[1]))
 214             database.FileTypes = database.FileType(patterns, order)
 215         else:
 216             croak('Unrecognized config line: "%s"' % (line))
 217         line = ReadConfigLine(file)
 218