misc/ini.py

   1 # Copyright (c) 2001, 2002, 2003 Python Software Foundation
   2 # Copyright (c) 2004 Paramjit Oberoi <param.cs.wisc.edu>
   3 # All Rights Reserved.  See LICENSE-PSF & LICENSE for details.
   4
   5 """Access and/or modify INI files
   6
   7 * Compatiable with ConfigParser
   8 * Preserves order of sections & options
   9 * Preserves comments/blank lines/etc
  10 * More convenient access to data
  11
  12 Example:
  13
  14     >>> from StringIO import StringIO
  15     >>> sio = StringIO('''# configure foo-application
  16     ... [foo]
  17     ... bar1 = qualia
  18     ... bar2 = 1977
  19     ... [foo-ext]
  20     ... special = 1''')
  21
  22     >>> cfg = INIConfig(sio)
  23     >>> print cfg.foo.bar1
  24     qualia
  25     >>> print cfg['foo-ext'].special
  26     1
  27     >>> cfg.foo.newopt = 'hi!'
  28
  29     >>> print cfg
  30     # configure foo-application
  31     [foo]
  32     bar1 = qualia
  33     bar2 = 1977
  34     newopt = hi!
  35     [foo-ext]
  36     special = 1
  37
  38 """
  39
  40 # An ini parser that supports ordered sections/options
  41 # Also supports updates, while preserving structure
  42 # Backward-compatiable with ConfigParser
  43
  44 import re
  45 from iniparse import config
  46 from sets import Set
  47 from ConfigParser import DEFAULTSECT, ParsingError, MissingSectionHeaderError
  48
  49 class LineType(object):
  50     line = None
  51
  52     def __init__(self, line=None):
  53         if line is not None:
  54             self.line = line.strip('\n')
  55
  56     # Return the original line for unmodified objects
  57     # Otherwise construct using the current attribute values
  58     def __str__(self):
  59         if self.line is not None:
  60             return self.line
  61         else:
  62             return self.to_string()
  63
  64     # If an attribute is modified after initialization
  65     # set line to None since it is no longer accurate.
  66     def __setattr__(self, name, value):
  67         if hasattr(self,name):
  68             self.__dict__['line'] = None
  69         self.__dict__[name] = value
  70
  71     def to_string(self):
  72         raise Exception('This method must be overridden in derived classes')
  73
  74
  75 class SectionLine(LineType):
  76     regex =  re.compile(r'^\['
  77                         r'(?P<name>[^]]+)'
  78                         r'\]\s*'
  79                         r'((?P<csep>;|#)(?P<comment>.*))?$')
  80
  81     def __init__(self, name, comment=None, comment_separator=None,
  82                              comment_offset=-1, line=None):
  83         super(SectionLine, self).__init__(line)
  84         self.name = name
  85         self.comment = comment
  86         self.comment_separator = comment_separator
  87         self.comment_offset = comment_offset
  88
  89     def to_string(self):
  90         out = '[' + self.name + ']'
  91         if self.comment is not None:
  92             # try to preserve indentation of comments
  93             out = (out+' ').ljust(self.comment_offset)
  94             out = out + self.comment_separator + self.comment
  95         return out
  96
  97     def parse(cls, line):
  98         m = cls.regex.match(line.rstrip())
  99         if m is None:
 100             return None
 101         return cls(m.group('name'), m.group('comment'),
 102                    m.group('csep'), m.start('csep'),
 103                    line)
 104     parse = classmethod(parse)
 105
 106
 107 class OptionLine(LineType):
 108     def __init__(self, name, value, separator=' = ', comment=None,
 109                  comment_separator=None, comment_offset=-1, line=None):
 110         super(OptionLine, self).__init__(line)
 111         self.name = name
 112         self.value = value
 113         self.separator = separator
 114         self.comment = comment
 115         self.comment_separator = comment_separator
 116         self.comment_offset = comment_offset
 117
 118     def to_string(self):
 119         out = '%s%s%s' % (self.name, self.separator, self.value)
 120         if self.comment is not None:
 121             # try to preserve indentation of comments
 122             out = (out+' ').ljust(self.comment_offset)
 123             out = out + self.comment_separator + self.comment
 124         return out
 125
 126     regex = re.compile(r'^(?P<name>[^:=\s[][^:=\s]*)'
 127                        r'(?P<sep>\s*[:=]\s*)'
 128                        r'(?P<value>.*)$')
 129
 130     def parse(cls, line):
 131         m = cls.regex.match(line.rstrip())
 132         if m is None:
 133             return None
 134
 135         name = m.group('name').rstrip()
 136         value = m.group('value')
 137         sep = m.group('sep')
 138
 139         # comments are not detected in the regex because
 140         # ensuring total compatibility with ConfigParser
 141         # requires that:
 142         #     option = value    ;comment   // value=='value'
 143         #     option = value;1  ;comment   // value=='value;1  ;comment'
 144         #
 145         # Doing this in a regex would be complicated.  I
 146         # think this is a bug.  The whole issue of how to
 147         # include ';' in the value needs to be addressed.
 148         # Also, '#' doesn't mark comments in options...
 149
 150         coff = value.find(';')
 151         if coff != -1 and value[coff-1].isspace():
 152             comment = value[coff+1:]
 153             csep = value[coff]
 154             value = value[:coff].rstrip()
 155             coff = m.start('value') + coff
 156         else:
 157             comment = None
 158             csep = None
 159             coff = -1
 160
 161         return cls(name, value, sep, comment, csep, coff, line)
 162     parse = classmethod(parse)
 163
 164
 165 class CommentLine(LineType):
 166     regex = re.compile(r'^(?P<csep>[;#]|[rR][eE][mM])'
 167                        r'(?P<comment>.*)$')
 168
 169     def __init__(self, comment='', separator='#', line=None):
 170         super(CommentLine, self).__init__(line)
 171         self.comment = comment
 172         self.separator = separator
 173
 174     def to_string(self):
 175         return self.separator + self.comment
 176
 177     def parse(cls, line):
 178         m = cls.regex.match(line.rstrip())
 179         if m is None:
 180             return None
 181         return cls(m.group('comment'), m.group('csep'), line)
 182     parse = classmethod(parse)
 183
 184
 185 class EmptyLine(LineType):
 186     # could make this a singleton
 187     def to_string(self):
 188         return ''
 189
 190     def parse(cls, line):
 191         if line.strip(): return None
 192         return cls(line)
 193     parse = classmethod(parse)
 194
 195
 196 class ContinuationLine(LineType):
 197     regex = re.compile(r'^\s+(?P<value>.*)$')
 198
 199     def __init__(self, value, value_offset=8, line=None):
 200         super(ContinuationLine, self).__init__(line)
 201         self.value = value
 202         self.value_offset = value_offset
 203
 204     def to_string(self):
 205         return ' '*self.value_offset + self.value
 206
 207     def parse(cls, line):
 208         m = cls.regex.match(line.rstrip())
 209         if m is None:
 210             return None
 211         return cls(m.group('value'), m.start('value'), line)
 212     parse = classmethod(parse)
 213
 214
 215 class LineContainer(object):
 216     def __init__(self, d=None):
 217         self.contents = []
 218         self.orgvalue = None
 219         if d:
 220             if isinstance(d, list): self.extend(d)
 221             else: self.add(d)
 222
 223     def add(self, x):
 224         self.contents.append(x)
 225
 226     def extend(self, x):
 227         for i in x: self.add(i)
 228
 229     def get_name(self):
 230         return self.contents[0].name
 231
 232     def set_name(self, data):
 233         self.contents[0].name = data
 234
 235     def get_value(self):
 236         if self.orgvalue is not None:
 237             return self.orgvalue
 238         elif len(self.contents) == 1:
 239             return self.contents[0].value
 240         else:
 241             return '\n'.join([str(x.value) for x in self.contents
 242                               if not isinstance(x, (CommentLine, EmptyLine))])
 243
 244     def set_value(self, data):
 245         self.orgvalue = data
 246         lines = str(data).split('\n')
 247         linediff = len(lines) - len(self.contents)
 248         if linediff > 0:
 249             for _ in range(linediff):
 250                 self.add(ContinuationLine(''))
 251         elif linediff < 0:
 252             self.contents = self.contents[:linediff]
 253         for i,v in enumerate(lines):
 254             self.contents[i].value = v
 255
 256     name = property(get_name, set_name)
 257     value = property(get_value, set_value)
 258
 259     def __str__(self):
 260         s = [str(x) for x in self.contents]
 261         return '\n'.join(s)
 262
 263     def finditer(self, key):
 264         for x in self.contents[::-1]:
 265             if hasattr(x, 'name') and x.name==key:
 266                 yield x
 267
 268     def find(self, key):
 269         for x in self.finditer(key):
 270             return x
 271         raise KeyError(key)
 272
 273
 274 def _make_xform_property(myattrname, srcattrname=None):
 275     private_attrname = myattrname + 'value'
 276     private_srcname = myattrname + 'source'
 277     if srcattrname is None:
 278         srcattrname = myattrname
 279
 280     def getfn(self):
 281         srcobj = getattr(self, private_srcname)
 282         if srcobj is not None:
 283             return getattr(srcobj, srcattrname)
 284         else:
 285             return getattr(self, private_attrname)
 286
 287     def setfn(self, value):
 288         srcobj = getattr(self, private_srcname)
 289         if srcobj is not None:
 290             setattr(srcobj, srcattrname, value)
 291         else:
 292             setattr(self, private_attrname, value)
 293
 294     return property(getfn, setfn)
 295
 296
 297 class INISection(config.ConfigNamespace):
 298     _lines = None
 299     _options = None
 300     _defaults = None
 301     _optionxformvalue = None
 302     _optionxformsource = None
 303     def __init__(self, lineobj, defaults = None,
 304                        optionxformvalue=None, optionxformsource=None):
 305         self._lines = [lineobj]
 306         self._defaults = defaults
 307         self._optionxformvalue = optionxformvalue
 308         self._optionxformsource = optionxformsource
 309         self._options = {}
 310
 311     _optionxform = _make_xform_property('_optionxform')
 312
 313     def __getitem__(self, key):
 314         if key == '__name__':
 315             return self._lines[-1].name
 316         if self._optionxform: key = self._optionxform(key)
 317         try:
 318             return self._options[key].value
 319         except KeyError:
 320             if self._defaults and key in self._defaults._options:
 321                 return self._defaults._options[key].value
 322             else:
 323                 raise
 324
 325     def __setitem__(self, key, value):
 326         if self._optionxform: xkey = self._optionxform(key)
 327         else: xkey = key
 328         if xkey not in self._options:
 329             # create a dummy object - value may have multiple lines
 330             obj = LineContainer(OptionLine(key, ''))
 331             self._lines[-1].add(obj)
 332             self._options[xkey] = obj
 333         # the set_value() function in LineContainer
 334         # automatically handles multi-line values
 335         self._options[xkey].value = value
 336
 337     def __delitem__(self, key):
 338         if self._optionxform: key = self._optionxform(key)
 339         for l in self._lines:
 340             remaining = []
 341             for o in l.contents:
 342                 if isinstance(o, LineContainer):
 343                     n = o.name
 344                     if self._optionxform: n = self._optionxform(n)
 345                     if key != n: remaining.append(o)
 346                 else:
 347                     remaining.append(o)
 348             l.contents = remaining
 349         del self._options[key]
 350
 351     def __iter__(self):
 352         d = Set()
 353         for l in self._lines:
 354             for x in l.contents:
 355                 if isinstance(x, LineContainer):
 356                     if self._optionxform:
 357                         ans = self._optionxform(x.name)
 358                     else:
 359                         ans = x.name
 360                     if ans not in d:
 361                         yield ans
 362                         d.add(ans)
 363         if self._defaults:
 364             for x in self._defaults:
 365                 if x not in d:
 366                     yield x
 367                     d.add(x)
 368
 369     def new_namespace(self, name):
 370         raise Exception('No sub-sections allowed', name)
 371
 372
 373 def make_comment(line):
 374     return CommentLine(line.rstrip())
 375
 376
 377 def readline_iterator(f):
 378     """iterate over a file by only using the file object's readline method"""
 379
 380     have_newline = False
 381     while True:
 382         line = f.readline()
 383
 384         if not line:
 385             if have_newline:
 386                 yield ""
 387             return
 388
 389         if line.endswith('\n'):
 390             have_newline = True
 391         else:
 392             have_newline = False
 393
 394         yield line
 395
 396
 397 class INIConfig(config.ConfigNamespace):
 398     _data = None
 399     _sections = None
 400     _defaults = None
 401     _optionxformvalue = None
 402     _optionxformsource = None
 403     _sectionxformvalue = None
 404     _sectionxformsource = None
 405     _parse_exc = None
 406     def __init__(self, fp=None, defaults = None, parse_exc=True,
 407                  optionxformvalue=str.lower, optionxformsource=None,
 408                  sectionxformvalue=None, sectionxformsource=None):
 409         self._data = LineContainer()
 410         self._parse_exc = parse_exc
 411         self._optionxformvalue = optionxformvalue
 412         self._optionxformsource = optionxformsource
 413         self._sectionxformvalue = sectionxformvalue
 414         self._sectionxformsource = sectionxformsource
 415         self._sections = {}
 416         if defaults is None: defaults = {}
 417         self._defaults = INISection(LineContainer(), optionxformsource=self)
 418         for name, value in defaults.iteritems():
 419             self._defaults[name] = value
 420         if fp is not None:
 421             self.readfp(fp)
 422
 423     _optionxform = _make_xform_property('_optionxform', 'optionxform')
 424     _sectionxform = _make_xform_property('_sectionxform', 'optionxform')
 425
 426     def __getitem__(self, key):
 427         if key == DEFAULTSECT:
 428             return self._defaults
 429         if self._sectionxform: key = self._sectionxform(key)
 430         return self._sections[key]
 431
 432     def __setitem__(self, key, value):
 433         raise Exception('Values must be inside sections', key, value)
 434
 435     def __delitem__(self, key):
 436         if self._sectionxform: key = self._sectionxform(key)
 437         for line in self._sections[key]._lines:
 438             self._data.contents.remove(line)
 439         del self._sections[key]
 440
 441     def __iter__(self):
 442         d = Set()
 443         for x in self._data.contents:
 444             if isinstance(x, LineContainer):
 445                 if x.name not in d:
 446                     yield x.name
 447                     d.add(x.name)
 448
 449     def new_namespace(self, name):
 450         if self._data.contents:
 451             self._data.add(EmptyLine())
 452         obj = LineContainer(SectionLine(name))
 453         self._data.add(obj)
 454         if self._sectionxform: name = self._sectionxform(name)
 455         if name in self._sections:
 456             ns = self._sections[name]
 457             ns._lines.append(obj)
 458         else:
 459             ns = INISection(obj, defaults=self._defaults,
 460                             optionxformsource=self)
 461             self._sections[name] = ns
 462         return ns
 463
 464     def __str__(self):
 465         return str(self._data)
 466
 467     _line_types = [EmptyLine, CommentLine,
 468                    SectionLine, OptionLine,
 469                    ContinuationLine]
 470
 471     def _parse(self, line):
 472         for linetype in self._line_types:
 473             lineobj = linetype.parse(line)
 474             if lineobj:
 475                 return lineobj
 476         else:
 477             # can't parse line
 478             return None
 479
 480     def readfp(self, fp):
 481         cur_section = None
 482         cur_option = None
 483         cur_section_name = None
 484         cur_option_name = None
 485         pending_lines = []
 486         try:
 487             fname = fp.name
 488         except AttributeError:
 489             fname = '<???>'
 490         linecount = 0
 491         exc = None
 492         line = None
 493
 494         for line in readline_iterator(fp):
 495             lineobj = self._parse(line)
 496             linecount += 1
 497
 498             if not cur_section and not isinstance(lineobj,
 499                                 (CommentLine, EmptyLine, SectionLine)):
 500                 if self._parse_exc:
 501                     raise MissingSectionHeaderError(fname, linecount, line)
 502                 else:
 503                     lineobj = make_comment(line)
 504
 505             if lineobj is None:
 506                 if self._parse_exc:
 507                     if exc is None: exc = ParsingError(fname)
 508                     exc.append(linecount, line)
 509                 lineobj = make_comment(line)
 510
 511             if isinstance(lineobj, ContinuationLine):
 512                 if cur_option:
 513                     cur_option.extend(pending_lines)
 514                     pending_lines = []
 515                     cur_option.add(lineobj)
 516                 else:
 517                     # illegal continuation line - convert to comment
 518                     if self._parse_exc:
 519                         if exc is None: exc = ParsingError(fname)
 520                         exc.append(linecount, line)
 521                     lineobj = make_comment(line)
 522
 523             if isinstance(lineobj, OptionLine):
 524                 cur_section.extend(pending_lines)
 525                 pending_lines = []
 526                 cur_option = LineContainer(lineobj)
 527                 cur_section.add(cur_option)
 528                 if self._optionxform:
 529                     cur_option_name = self._optionxform(cur_option.name)
 530                 else:
 531                     cur_option_name = cur_option.name
 532                 if cur_section_name == DEFAULTSECT:
 533                     optobj = self._defaults
 534                 else:
 535                     optobj = self._sections[cur_section_name]
 536                 optobj._options[cur_option_name] = cur_option
 537
 538             if isinstance(lineobj, SectionLine):
 539                 self._data.extend(pending_lines)
 540                 pending_lines = []
 541                 cur_section = LineContainer(lineobj)
 542                 self._data.add(cur_section)
 543                 cur_option = None
 544                 cur_option_name = None
 545                 if cur_section.name == DEFAULTSECT:
 546                     self._defaults._lines.append(cur_section)
 547                     cur_section_name = DEFAULTSECT
 548                 else:
 549                     if self._sectionxform:
 550                         cur_section_name = self._sectionxform(cur_section.name)
 551                     else:
 552                         cur_section_name = cur_section.name
 553                     if not self._sections.has_key(cur_section_name):
 554                         self._sections[cur_section_name] = \
 555                                 INISection(cur_section, defaults=self._defaults,
 556                                            optionxformsource=self)
 557                     else:
 558                         self._sections[cur_section_name]._lines.append(cur_section)
 559
 560             if isinstance(lineobj, (CommentLine, EmptyLine)):
 561                 pending_lines.append(lineobj)
 562
 563         self._data.extend(pending_lines)
 564         if line and line[-1]=='\n':
 565             self._data.add(EmptyLine())
 566
 567         if exc:
 568             raise exc
 569