Lib/mailbox.py

   1 #! /usr/bin/env python
   2
   3 """Read/write support for Maildir, mbox, MH, Babyl, and MMDF mailboxes."""
   4
   5 import sys
   6 import os
   7 import time
   8 import calendar
   9 import socket
  10 import errno
  11 import copy
  12 import email
  13 import email.Message
  14 import email.Generator
  15 import rfc822
  16 import StringIO
  17 try:
  18     if sys.platform == 'os2emx':
  19         # OS/2 EMX fcntl() not adequate
  20         raise ImportError
  21     import fcntl
  22 except ImportError:
  23     fcntl = None
  24
  25 __all__ = [ 'Mailbox', 'Maildir', 'mbox', 'MH', 'Babyl', 'MMDF',
  26             'Message', 'MaildirMessage', 'mboxMessage', 'MHMessage',
  27             'BabylMessage', 'MMDFMessage', 'UnixMailbox',
  28             'PortableUnixMailbox', 'MmdfMailbox', 'MHMailbox', 'BabylMailbox' ]
  29
  30 class Mailbox:
  31     """A group of messages in a particular place."""
  32
  33     def __init__(self, path, factory=None, create=True):
  34         """Initialize a Mailbox instance."""
  35         self._path = os.path.abspath(os.path.expanduser(path))
  36         self._factory = factory
  37
  38     def add(self, message):
  39         """Add message and return assigned key."""
  40         raise NotImplementedError('Method must be implemented by subclass')
  41
  42     def remove(self, key):
  43         """Remove the keyed message; raise KeyError if it doesn't exist."""
  44         raise NotImplementedError('Method must be implemented by subclass')
  45
  46     def __delitem__(self, key):
  47         self.remove(key)
  48
  49     def discard(self, key):
  50         """If the keyed message exists, remove it."""
  51         try:
  52             self.remove(key)
  53         except KeyError:
  54             pass
  55
  56     def __setitem__(self, key, message):
  57         """Replace the keyed message; raise KeyError if it doesn't exist."""
  58         raise NotImplementedError('Method must be implemented by subclass')
  59
  60     def get(self, key, default=None):
  61         """Return the keyed message, or default if it doesn't exist."""
  62         try:
  63             return self.__getitem__(key)
  64         except KeyError:
  65             return default
  66
  67     def __getitem__(self, key):
  68         """Return the keyed message; raise KeyError if it doesn't exist."""
  69         if not self._factory:
  70             return self.get_message(key)
  71         else:
  72             return self._factory(self.get_file(key))
  73
  74     def get_message(self, key):
  75         """Return a Message representation or raise a KeyError."""
  76         raise NotImplementedError('Method must be implemented by subclass')
  77
  78     def get_string(self, key):
  79         """Return a string representation or raise a KeyError."""
  80         raise NotImplementedError('Method must be implemented by subclass')
  81
  82     def get_file(self, key):
  83         """Return a file-like representation or raise a KeyError."""
  84         raise NotImplementedError('Method must be implemented by subclass')
  85
  86     def iterkeys(self):
  87         """Return an iterator over keys."""
  88         raise NotImplementedError('Method must be implemented by subclass')
  89
  90     def keys(self):
  91         """Return a list of keys."""
  92         return list(self.iterkeys())
  93
  94     def itervalues(self):
  95         """Return an iterator over all messages."""
  96         for key in self.iterkeys():
  97             try:
  98                 value = self[key]
  99             except KeyError:
 100                 continue
 101             yield value
 102
 103     def __iter__(self):
 104         return self.itervalues()
 105
 106     def values(self):
 107         """Return a list of messages. Memory intensive."""
 108         return list(self.itervalues())
 109
 110     def iteritems(self):
 111         """Return an iterator over (key, message) tuples."""
 112         for key in self.iterkeys():
 113             try:
 114                 value = self[key]
 115             except KeyError:
 116                 continue
 117             yield (key, value)
 118
 119     def items(self):
 120         """Return a list of (key, message) tuples. Memory intensive."""
 121         return list(self.iteritems())
 122
 123     def has_key(self, key):
 124         """Return True if the keyed message exists, False otherwise."""
 125         raise NotImplementedError('Method must be implemented by subclass')
 126
 127     def __contains__(self, key):
 128         return self.has_key(key)
 129
 130     def __len__(self):
 131         """Return a count of messages in the mailbox."""
 132         raise NotImplementedError('Method must be implemented by subclass')
 133
 134     def clear(self):
 135         """Delete all messages."""
 136         for key in self.iterkeys():
 137             self.discard(key)
 138
 139     def pop(self, key, default=None):
 140         """Delete the keyed message and return it, or default."""
 141         try:
 142             result = self[key]
 143         except KeyError:
 144             return default
 145         self.discard(key)
 146         return result
 147
 148     def popitem(self):
 149         """Delete an arbitrary (key, message) pair and return it."""
 150         for key in self.iterkeys():
 151             return (key, self.pop(key))     # This is only run once.
 152         else:
 153             raise KeyError('No messages in mailbox')
 154
 155     def update(self, arg=None):
 156         """Change the messages that correspond to certain keys."""
 157         if hasattr(arg, 'iteritems'):
 158             source = arg.iteritems()
 159         elif hasattr(arg, 'items'):
 160             source = arg.items()
 161         else:
 162             source = arg
 163         bad_key = False
 164         for key, message in source:
 165             try:
 166                 self[key] = message
 167             except KeyError:
 168                 bad_key = True
 169         if bad_key:
 170             raise KeyError('No message with key(s)')
 171
 172     def flush(self):
 173         """Write any pending changes to the disk."""
 174         raise NotImplementedError('Method must be implemented by subclass')
 175
 176     def lock(self):
 177         """Lock the mailbox."""
 178         raise NotImplementedError('Method must be implemented by subclass')
 179
 180     def unlock(self):
 181         """Unlock the mailbox if it is locked."""
 182         raise NotImplementedError('Method must be implemented by subclass')
 183
 184     def close(self):
 185         """Flush and close the mailbox."""
 186         raise NotImplementedError('Method must be implemented by subclass')
 187
 188     def _dump_message(self, message, target, mangle_from_=False):
 189         # Most files are opened in binary mode to allow predictable seeking.
 190         # To get native line endings on disk, the user-friendly \n line endings
 191         # used in strings and by email.Message are translated here.
 192         """Dump message contents to target file."""
 193         if isinstance(message, email.Message.Message):
 194             buffer = StringIO.StringIO()
 195             gen = email.Generator.Generator(buffer, mangle_from_, 0)
 196             gen.flatten(message)
 197             buffer.seek(0)
 198             target.write(buffer.read().replace('\n', os.linesep))
 199         elif isinstance(message, str):
 200             if mangle_from_:
 201                 message = message.replace('\nFrom ', '\n>From ')
 202             message = message.replace('\n', os.linesep)
 203             target.write(message)
 204         elif hasattr(message, 'read'):
 205             while True:
 206                 line = message.readline()
 207                 if line == '':
 208                     break
 209                 if mangle_from_ and line.startswith('From '):
 210                     line = '>From ' + line[5:]
 211                 line = line.replace('\n', os.linesep)
 212                 target.write(line)
 213         else:
 214             raise TypeError('Invalid message type: %s' % type(message))
 215
 216
 217 class Maildir(Mailbox):
 218     """A qmail-style Maildir mailbox."""
 219
 220     colon = ':'
 221
 222     def __init__(self, dirname, factory=rfc822.Message, create=True):
 223         """Initialize a Maildir instance."""
 224         Mailbox.__init__(self, dirname, factory, create)
 225         if not os.path.exists(self._path):
 226             if create:
 227                 os.mkdir(self._path, 0700)
 228                 os.mkdir(os.path.join(self._path, 'tmp'), 0700)
 229                 os.mkdir(os.path.join(self._path, 'new'), 0700)
 230                 os.mkdir(os.path.join(self._path, 'cur'), 0700)
 231             else:
 232                 raise NoSuchMailboxError(self._path)
 233         self._toc = {}
 234
 235     def add(self, message):
 236         """Add message and return assigned key."""
 237         tmp_file = self._create_tmp()
 238         try:
 239             self._dump_message(message, tmp_file)
 240         finally:
 241             tmp_file.close()
 242         if isinstance(message, MaildirMessage):
 243             subdir = message.get_subdir()
 244             suffix = self.colon + message.get_info()
 245             if suffix == self.colon:
 246                 suffix = ''
 247         else:
 248             subdir = 'new'
 249             suffix = ''
 250         uniq = os.path.basename(tmp_file.name).split(self.colon)[0]
 251         dest = os.path.join(self._path, subdir, uniq + suffix)
 252         os.rename(tmp_file.name, dest)
 253         if isinstance(message, MaildirMessage):
 254             os.utime(dest, (os.path.getatime(dest), message.get_date()))
 255         return uniq
 256
 257     def remove(self, key):
 258         """Remove the keyed message; raise KeyError if it doesn't exist."""
 259         os.remove(os.path.join(self._path, self._lookup(key)))
 260
 261     def discard(self, key):
 262         """If the keyed message exists, remove it."""
 263         # This overrides an inapplicable implementation in the superclass.
 264         try:
 265             self.remove(key)
 266         except KeyError:
 267             pass
 268         except OSError, e:
 269             if e.errno != errno.ENOENT:
 270                 raise
 271
 272     def __setitem__(self, key, message):
 273         """Replace the keyed message; raise KeyError if it doesn't exist."""
 274         old_subpath = self._lookup(key)
 275         temp_key = self.add(message)
 276         temp_subpath = self._lookup(temp_key)
 277         if isinstance(message, MaildirMessage):
 278             # temp's subdir and suffix were specified by message.
 279             dominant_subpath = temp_subpath
 280         else:
 281             # temp's subdir and suffix were defaults from add().
 282             dominant_subpath = old_subpath
 283         subdir = os.path.dirname(dominant_subpath)
 284         if self.colon in dominant_subpath:
 285             suffix = self.colon + dominant_subpath.split(self.colon)[-1]
 286         else:
 287             suffix = ''
 288         self.discard(key)
 289         new_path = os.path.join(self._path, subdir, key + suffix)
 290         os.rename(os.path.join(self._path, temp_subpath), new_path)
 291         if isinstance(message, MaildirMessage):
 292             os.utime(new_path, (os.path.getatime(new_path),
 293                                 message.get_date()))
 294
 295     def get_message(self, key):
 296         """Return a Message representation or raise a KeyError."""
 297         subpath = self._lookup(key)
 298         f = open(os.path.join(self._path, subpath), 'r')
 299         try:
 300             msg = MaildirMessage(f)
 301         finally:
 302             f.close()
 303         subdir, name = os.path.split(subpath)
 304         msg.set_subdir(subdir)
 305         if self.colon in name:
 306             msg.set_info(name.split(self.colon)[-1])
 307         msg.set_date(os.path.getmtime(os.path.join(self._path, subpath)))
 308         return msg
 309
 310     def get_string(self, key):
 311         """Return a string representation or raise a KeyError."""
 312         f = open(os.path.join(self._path, self._lookup(key)), 'r')
 313         try:
 314             return f.read()
 315         finally:
 316             f.close()
 317
 318     def get_file(self, key):
 319         """Return a file-like representation or raise a KeyError."""
 320         f = open(os.path.join(self._path, self._lookup(key)), 'rb')
 321         return _ProxyFile(f)
 322
 323     def iterkeys(self):
 324         """Return an iterator over keys."""
 325         self._refresh()
 326         for key in self._toc:
 327             try:
 328                 self._lookup(key)
 329             except KeyError:
 330                 continue
 331             yield key
 332
 333     def has_key(self, key):
 334         """Return True if the keyed message exists, False otherwise."""
 335         self._refresh()
 336         return key in self._toc
 337
 338     def __len__(self):
 339         """Return a count of messages in the mailbox."""
 340         self._refresh()
 341         return len(self._toc)
 342
 343     def flush(self):
 344         """Write any pending changes to disk."""
 345         return  # Maildir changes are always written immediately.
 346
 347     def lock(self):
 348         """Lock the mailbox."""
 349         return
 350
 351     def unlock(self):
 352         """Unlock the mailbox if it is locked."""
 353         return
 354
 355     def close(self):
 356         """Flush and close the mailbox."""
 357         return
 358
 359     def list_folders(self):
 360         """Return a list of folder names."""
 361         result = []
 362         for entry in os.listdir(self._path):
 363             if len(entry) > 1 and entry[0] == '.' and \
 364                os.path.isdir(os.path.join(self._path, entry)):
 365                 result.append(entry[1:])
 366         return result
 367
 368     def get_folder(self, folder):
 369         """Return a Maildir instance for the named folder."""
 370         return Maildir(os.path.join(self._path, '.' + folder), create=False)
 371
 372     def add_folder(self, folder):
 373         """Create a folder and return a Maildir instance representing it."""
 374         path = os.path.join(self._path, '.' + folder)
 375         result = Maildir(path)
 376         maildirfolder_path = os.path.join(path, 'maildirfolder')
 377         if not os.path.exists(maildirfolder_path):
 378             os.close(os.open(maildirfolder_path, os.O_CREAT | os.O_WRONLY))
 379         return result
 380
 381     def remove_folder(self, folder):
 382         """Delete the named folder, which must be empty."""
 383         path = os.path.join(self._path, '.' + folder)
 384         for entry in os.listdir(os.path.join(path, 'new')) + \
 385                      os.listdir(os.path.join(path, 'cur')):
 386             if len(entry) < 1 or entry[0] != '.':
 387                 raise NotEmptyError('Folder contains message(s): %s' % folder)
 388         for entry in os.listdir(path):
 389             if entry != 'new' and entry != 'cur' and entry != 'tmp' and \
 390                os.path.isdir(os.path.join(path, entry)):
 391                 raise NotEmptyError("Folder contains subdirectory '%s': %s" %
 392                                     (folder, entry))
 393         for root, dirs, files in os.walk(path, topdown=False):
 394             for entry in files:
 395                 os.remove(os.path.join(root, entry))
 396             for entry in dirs:
 397                 os.rmdir(os.path.join(root, entry))
 398         os.rmdir(path)
 399
 400     def clean(self):
 401         """Delete old files in "tmp"."""
 402         now = time.time()
 403         for entry in os.listdir(os.path.join(self._path, 'tmp')):
 404             path = os.path.join(self._path, 'tmp', entry)
 405             if now - os.path.getatime(path) > 129600:   # 60 * 60 * 36
 406                 os.remove(path)
 407
 408     _count = 1  # This is used to generate unique file names.
 409
 410     def _create_tmp(self):
 411         """Create a file in the tmp subdirectory and open and return it."""
 412         now = time.time()
 413         hostname = socket.gethostname()
 414         if '/' in hostname:
 415             hostname = hostname.replace('/', r'\057')
 416         if ':' in hostname:
 417             hostname = hostname.replace(':', r'\072')
 418         uniq = "%s.M%sP%sQ%s.%s" % (int(now), int(now % 1 * 1e6), os.getpid(),
 419                                     Maildir._count, hostname)
 420         path = os.path.join(self._path, 'tmp', uniq)
 421         try:
 422             os.stat(path)
 423         except OSError, e:
 424             if e.errno == errno.ENOENT:
 425                 Maildir._count += 1
 426                 return open(path, 'wb+')
 427             else:
 428                 raise
 429         else:
 430             raise ExternalClashError('Name clash prevented file creation: %s' %
 431                                      path)
 432
 433     def _refresh(self):
 434         """Update table of contents mapping."""
 435         self._toc = {}
 436         for subdir in ('new', 'cur'):
 437             for entry in os.listdir(os.path.join(self._path, subdir)):
 438                 uniq = entry.split(self.colon)[0]
 439                 self._toc[uniq] = os.path.join(subdir, entry)
 440
 441     def _lookup(self, key):
 442         """Use TOC to return subpath for given key, or raise a KeyError."""
 443         try:
 444             if os.path.exists(os.path.join(self._path, self._toc[key])):
 445                 return self._toc[key]
 446         except KeyError:
 447             pass
 448         self._refresh()
 449         try:
 450             return self._toc[key]
 451         except KeyError:
 452             raise KeyError('No message with key: %s' % key)
 453
 454     # This method is for backward compatibility only.
 455     def next(self):
 456         """Return the next message in a one-time iteration."""
 457         if not hasattr(self, '_onetime_keys'):
 458             self._onetime_keys = self.iterkeys()
 459         while True:
 460             try:
 461                 return self[self._onetime_keys.next()]
 462             except StopIteration:
 463                 return None
 464             except KeyError:
 465                 continue
 466
 467
 468 class _singlefileMailbox(Mailbox):
 469     """A single-file mailbox."""
 470
 471     def __init__(self, path, factory=None, create=True):
 472         """Initialize a single-file mailbox."""
 473         Mailbox.__init__(self, path, factory, create)
 474         try:
 475             f = open(self._path, 'rb+')
 476         except IOError, e:
 477             if e.errno == errno.ENOENT:
 478                 if create:
 479                     f = open(self._path, 'wb+')
 480                 else:
 481                     raise NoSuchMailboxError(self._path)
 482             elif e.errno == errno.EACCES:
 483                 f = open(self._path, 'rb')
 484             else:
 485                 raise
 486         self._file = f
 487         self._toc = None
 488         self._next_key = 0
 489         self._pending = False   # No changes require rewriting the file.
 490         self._locked = False
 491
 492     def add(self, message):
 493         """Add message and return assigned key."""
 494         self._lookup()
 495         self._toc[self._next_key] = self._append_message(message)
 496         self._next_key += 1
 497         self._pending = True
 498         return self._next_key - 1
 499
 500     def remove(self, key):
 501         """Remove the keyed message; raise KeyError if it doesn't exist."""
 502         self._lookup(key)
 503         del self._toc[key]
 504         self._pending = True
 505
 506     def __setitem__(self, key, message):
 507         """Replace the keyed message; raise KeyError if it doesn't exist."""
 508         self._lookup(key)
 509         self._toc[key] = self._append_message(message)
 510         self._pending = True
 511
 512     def iterkeys(self):
 513         """Return an iterator over keys."""
 514         self._lookup()
 515         for key in self._toc.keys():
 516             yield key
 517
 518     def has_key(self, key):
 519         """Return True if the keyed message exists, False otherwise."""
 520         self._lookup()
 521         return key in self._toc
 522
 523     def __len__(self):
 524         """Return a count of messages in the mailbox."""
 525         self._lookup()
 526         return len(self._toc)
 527
 528     def lock(self):
 529         """Lock the mailbox."""
 530         if not self._locked:
 531             _lock_file(self._file)
 532             self._locked = True
 533
 534     def unlock(self):
 535         """Unlock the mailbox if it is locked."""
 536         if self._locked:
 537             _unlock_file(self._file)
 538             self._locked = False
 539
 540     def flush(self):
 541         """Write any pending changes to disk."""
 542         if not self._pending:
 543             return
 544         self._lookup()
 545         new_file = _create_temporary(self._path)
 546         try:
 547             new_toc = {}
 548             self._pre_mailbox_hook(new_file)
 549             for key in sorted(self._toc.keys()):
 550                 start, stop = self._toc[key]
 551                 self._file.seek(start)
 552                 self._pre_message_hook(new_file)
 553                 new_start = new_file.tell()
 554                 while True:
 555                     buffer = self._file.read(min(4096,
 556                                                  stop - self._file.tell()))
 557                     if buffer == '':
 558                         break
 559                     new_file.write(buffer)
 560                 new_toc[key] = (new_start, new_file.tell())
 561                 self._post_message_hook(new_file)
 562         except:
 563             new_file.close()
 564             os.remove(new_file.name)
 565             raise
 566         new_file.close()
 567         self._file.close()
 568         try:
 569             os.rename(new_file.name, self._path)
 570         except OSError, e:
 571             if e.errno == errno.EEXIST or \
 572               (os.name == 'os2' and e.errno == errno.EACCES):
 573                 os.remove(self._path)
 574                 os.rename(new_file.name, self._path)
 575             else:
 576                 raise
 577         self._file = open(self._path, 'rb+')
 578         self._toc = new_toc
 579         self._pending = False
 580         if self._locked:
 581             _lock_file(new_file, dotlock=False)
 582
 583     def _pre_mailbox_hook(self, f):
 584         """Called before writing the mailbox to file f."""
 585         return
 586
 587     def _pre_message_hook(self, f):
 588         """Called before writing each message to file f."""
 589         return
 590
 591     def _post_message_hook(self, f):
 592         """Called after writing each message to file f."""
 593         return
 594
 595     def close(self):
 596         """Flush and close the mailbox."""
 597         self.flush()
 598         if self._locked:
 599             self.unlock()
 600         self._file.close()
 601
 602     def _lookup(self, key=None):
 603         """Return (start, stop) or raise KeyError."""
 604         if self._toc is None:
 605             self._generate_toc()
 606         if key is not None:
 607             try:
 608                 return self._toc[key]
 609             except KeyError:
 610                 raise KeyError('No message with key: %s' % key)
 611
 612     def _append_message(self, message):
 613         """Append message to mailbox and return (start, stop) offsets."""
 614         self._file.seek(0, 2)
 615         self._pre_message_hook(self._file)
 616         offsets = self._install_message(message)
 617         self._post_message_hook(self._file)
 618         self._file.flush()
 619         return offsets
 620
 621
 622
 623 class _mboxMMDF(_singlefileMailbox):
 624     """An mbox or MMDF mailbox."""
 625
 626     _mangle_from_ = True
 627
 628     def get_message(self, key):
 629         """Return a Message representation or raise a KeyError."""
 630         start, stop = self._lookup(key)
 631         self._file.seek(start)
 632         from_line = self._file.readline().replace(os.linesep, '')
 633         string = self._file.read(stop - self._file.tell())
 634         msg = self._message_factory(string.replace(os.linesep, '\n'))
 635         msg.set_from(from_line[5:])
 636         return msg
 637
 638     def get_string(self, key, from_=False):
 639         """Return a string representation or raise a KeyError."""
 640         start, stop = self._lookup(key)
 641         self._file.seek(start)
 642         if not from_:
 643             self._file.readline()
 644         string = self._file.read(stop - self._file.tell())
 645         return string.replace(os.linesep, '\n')
 646
 647     def get_file(self, key, from_=False):
 648         """Return a file-like representation or raise a KeyError."""
 649         start, stop = self._lookup(key)
 650         self._file.seek(start)
 651         if not from_:
 652             self._file.readline()
 653         return _PartialFile(self._file, self._file.tell(), stop)
 654
 655     def _install_message(self, message):
 656         """Format a message and blindly write to self._file."""
 657         from_line = None
 658         if isinstance(message, str) and message.startswith('From '):
 659             newline = message.find('\n')
 660             if newline != -1:
 661                 from_line = message[:newline]
 662                 message = message[newline + 1:]
 663             else:
 664                 from_line = message
 665                 message = ''
 666         elif isinstance(message, _mboxMMDFMessage):
 667             from_line = 'From ' + message.get_from()
 668         elif isinstance(message, email.Message.Message):
 669             from_line = message.get_unixfrom()  # May be None.
 670         if from_line is None:
 671             from_line = 'From MAILER-DAEMON %s' % time.asctime(time.gmtime())
 672         start = self._file.tell()
 673         self._file.write(from_line + os.linesep)
 674         self._dump_message(message, self._file, self._mangle_from_)
 675         stop = self._file.tell()
 676         return (start, stop)
 677
 678
 679 class mbox(_mboxMMDF):
 680     """A classic mbox mailbox."""
 681
 682     _mangle_from_ = True
 683
 684     def __init__(self, path, factory=None, create=True):
 685         """Initialize an mbox mailbox."""
 686         self._message_factory = mboxMessage
 687         _mboxMMDF.__init__(self, path, factory, create)
 688
 689     def _pre_message_hook(self, f):
 690         """Called before writing each message to file f."""
 691         if f.tell() != 0:
 692             f.write(os.linesep)
 693
 694     def _generate_toc(self):
 695         """Generate key-to-(start, stop) table of contents."""
 696         starts, stops = [], []
 697         self._file.seek(0)
 698         while True:
 699             line_pos = self._file.tell()
 700             line = self._file.readline()
 701             if line.startswith('From '):
 702                 if len(stops) < len(starts):
 703                     stops.append(line_pos - len(os.linesep))
 704                 starts.append(line_pos)
 705             elif line == '':
 706                 stops.append(line_pos)
 707                 break
 708         self._toc = dict(enumerate(zip(starts, stops)))
 709         self._next_key = len(self._toc)
 710
 711
 712 class MMDF(_mboxMMDF):
 713     """An MMDF mailbox."""
 714
 715     def __init__(self, path, factory=None, create=True):
 716         """Initialize an MMDF mailbox."""
 717         self._message_factory = MMDFMessage
 718         _mboxMMDF.__init__(self, path, factory, create)
 719
 720     def _pre_message_hook(self, f):
 721         """Called before writing each message to file f."""
 722         f.write('\001\001\001\001' + os.linesep)
 723
 724     def _post_message_hook(self, f):
 725         """Called after writing each message to file f."""
 726         f.write(os.linesep + '\001\001\001\001' + os.linesep)
 727
 728     def _generate_toc(self):
 729         """Generate key-to-(start, stop) table of contents."""
 730         starts, stops = [], []
 731         self._file.seek(0)
 732         next_pos = 0
 733         while True:
 734             line_pos = next_pos
 735             line = self._file.readline()
 736             next_pos = self._file.tell()
 737             if line.startswith('\001\001\001\001' + os.linesep):
 738                 starts.append(next_pos)
 739                 while True:
 740                     line_pos = next_pos
 741                     line = self._file.readline()
 742                     next_pos = self._file.tell()
 743                     if line == '\001\001\001\001' + os.linesep:
 744                         stops.append(line_pos - len(os.linesep))
 745                         break
 746                     elif line == '':
 747                         stops.append(line_pos)
 748                         break
 749             elif line == '':
 750                 break
 751         self._toc = dict(enumerate(zip(starts, stops)))
 752         self._next_key = len(self._toc)
 753
 754
 755 class MH(Mailbox):
 756     """An MH mailbox."""
 757
 758     def __init__(self, path, factory=None, create=True):
 759         """Initialize an MH instance."""
 760         Mailbox.__init__(self, path, factory, create)
 761         if not os.path.exists(self._path):
 762             if create:
 763                 os.mkdir(self._path, 0700)
 764                 os.close(os.open(os.path.join(self._path, '.mh_sequences'),
 765                                  os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0600))
 766             else:
 767                 raise NoSuchMailboxError(self._path)
 768         self._locked = False
 769
 770     def add(self, message):
 771         """Add message and return assigned key."""
 772         keys = self.keys()
 773         if len(keys) == 0:
 774             new_key = 1
 775         else:
 776             new_key = max(keys) + 1
 777         new_path = os.path.join(self._path, str(new_key))
 778         f = _create_carefully(new_path)
 779         try:
 780             if self._locked:
 781                 _lock_file(f)
 782             try:
 783                 self._dump_message(message, f)
 784                 if isinstance(message, MHMessage):
 785                     self._dump_sequences(message, new_key)
 786             finally:
 787                 if self._locked:
 788                     _unlock_file(f)
 789         finally:
 790             f.close()
 791         return new_key
 792
 793     def remove(self, key):
 794         """Remove the keyed message; raise KeyError if it doesn't exist."""
 795         path = os.path.join(self._path, str(key))
 796         try:
 797             f = open(path, 'rb+')
 798         except IOError, e:
 799             if e.errno == errno.ENOENT:
 800                 raise KeyError('No message with key: %s' % key)
 801             else:
 802                 raise
 803         try:
 804             if self._locked:
 805                 _lock_file(f)
 806             try:
 807                 f.close()
 808                 os.remove(os.path.join(self._path, str(key)))
 809             finally:
 810                 if self._locked:
 811                     _unlock_file(f)
 812         finally:
 813             f.close()
 814
 815     def __setitem__(self, key, message):
 816         """Replace the keyed message; raise KeyError if it doesn't exist."""
 817         path = os.path.join(self._path, str(key))
 818         try:
 819             f = open(path, 'rb+')
 820         except IOError, e:
 821             if e.errno == errno.ENOENT:
 822                 raise KeyError('No message with key: %s' % key)
 823             else:
 824                 raise
 825         try:
 826             if self._locked:
 827                 _lock_file(f)
 828             try:
 829                 os.close(os.open(path, os.O_WRONLY | os.O_TRUNC))
 830                 self._dump_message(message, f)
 831                 if isinstance(message, MHMessage):
 832                     self._dump_sequences(message, key)
 833             finally:
 834                 if self._locked:
 835                     _unlock_file(f)
 836         finally:
 837             f.close()
 838
 839     def get_message(self, key):
 840         """Return a Message representation or raise a KeyError."""
 841         try:
 842             if self._locked:
 843                 f = open(os.path.join(self._path, str(key)), 'r+')
 844             else:
 845                 f = open(os.path.join(self._path, str(key)), 'r')
 846         except IOError, e:
 847             if e.errno == errno.ENOENT:
 848                 raise KeyError('No message with key: %s' % key)
 849             else:
 850                 raise
 851         try:
 852             if self._locked:
 853                 _lock_file(f)
 854             try:
 855                 msg = MHMessage(f)
 856             finally:
 857                 if self._locked:
 858                     _unlock_file(f)
 859         finally:
 860             f.close()
 861         for name, key_list in self.get_sequences():
 862             if key in key_list:
 863                 msg.add_sequence(name)
 864         return msg
 865
 866     def get_string(self, key):
 867         """Return a string representation or raise a KeyError."""
 868         try:
 869             if self._locked:
 870                 f = open(os.path.join(self._path, str(key)), 'r+')
 871             else:
 872                 f = open(os.path.join(self._path, str(key)), 'r')
 873         except IOError, e:
 874             if e.errno == errno.ENOENT:
 875                 raise KeyError('No message with key: %s' % key)
 876             else:
 877                 raise
 878         try:
 879             if self._locked:
 880                 _lock_file(f)
 881             try:
 882                 return f.read()
 883             finally:
 884                 if self._locked:
 885                     _unlock_file(f)
 886         finally:
 887             f.close()
 888
 889     def get_file(self, key):
 890         """Return a file-like representation or raise a KeyError."""
 891         try:
 892             f = open(os.path.join(self._path, str(key)), 'rb')
 893         except IOError, e:
 894             if e.errno == errno.ENOENT:
 895                 raise KeyError('No message with key: %s' % key)
 896             else:
 897                 raise
 898         return _ProxyFile(f)
 899
 900     def iterkeys(self):
 901         """Return an iterator over keys."""
 902         return iter(sorted(int(entry) for entry in os.listdir(self._path)
 903                                       if entry.isdigit()))
 904
 905     def has_key(self, key):
 906         """Return True if the keyed message exists, False otherwise."""
 907         return os.path.exists(os.path.join(self._path, str(key)))
 908
 909     def __len__(self):
 910         """Return a count of messages in the mailbox."""
 911         return len(list(self.iterkeys()))
 912
 913     def lock(self):
 914         """Lock the mailbox."""
 915         if not self._locked:
 916             self._file = open(os.path.join(self._path, '.mh_sequences'), 'rb+')
 917             _lock_file(self._file)
 918             self._locked = True
 919
 920     def unlock(self):
 921         """Unlock the mailbox if it is locked."""
 922         if self._locked:
 923             _unlock_file(self._file)
 924             self._file.close()
 925             del self._file
 926             self._locked = False
 927
 928     def flush(self):
 929         """Write any pending changes to the disk."""
 930         return
 931
 932     def close(self):
 933         """Flush and close the mailbox."""
 934         if self._locked:
 935             self.unlock()
 936
 937     def list_folders(self):
 938         """Return a list of folder names."""
 939         result = []
 940         for entry in os.listdir(self._path):
 941             if os.path.isdir(os.path.join(self._path, entry)):
 942                 result.append(entry)
 943         return result
 944
 945     def get_folder(self, folder):
 946         """Return an MH instance for the named folder."""
 947         return MH(os.path.join(self._path, folder), create=False)
 948
 949     def add_folder(self, folder):
 950         """Create a folder and return an MH instance representing it."""
 951         return MH(os.path.join(self._path, folder))
 952
 953     def remove_folder(self, folder):
 954         """Delete the named folder, which must be empty."""
 955         path = os.path.join(self._path, folder)
 956         entries = os.listdir(path)
 957         if entries == ['.mh_sequences']:
 958             os.remove(os.path.join(path, '.mh_sequences'))
 959         elif entries == []:
 960             pass
 961         else:
 962             raise NotEmptyError('Folder not empty: %s' % self._path)
 963         os.rmdir(path)
 964
 965     def get_sequences(self):
 966         """Return a name-to-key-list dictionary to define each sequence."""
 967         results = {}
 968         f = open(os.path.join(self._path, '.mh_sequences'), 'r')
 969         try:
 970             all_keys = set(self.keys())
 971             for line in f:
 972                 try:
 973                     name, contents = line.split(':')
 974                     keys = set()
 975                     for spec in contents.split():
 976                         if spec.isdigit():
 977                             keys.add(int(spec))
 978                         else:
 979                             start, stop = (int(x) for x in spec.split('-'))
 980                             keys.update(range(start, stop + 1))
 981                     results[name] = [key for key in sorted(keys) \
 982                                          if key in all_keys]
 983                     if len(results[name]) == 0:
 984                         del results[name]
 985                 except ValueError:
 986                     raise FormatError('Invalid sequence specification: %s' %
 987                                       line.rstrip())
 988         finally:
 989             f.close()
 990         return results
 991
 992     def set_sequences(self, sequences):
 993         """Set sequences using the given name-to-key-list dictionary."""
 994         f = open(os.path.join(self._path, '.mh_sequences'), 'r+')
 995         try:
 996             os.close(os.open(f.name, os.O_WRONLY | os.O_TRUNC))
 997             for name, keys in sequences.iteritems():
 998                 if len(keys) == 0:
 999                     continue
1000                 f.write('%s:' % name)
1001                 prev = None
1002                 completing = False
1003                 for key in sorted(set(keys)):
1004                     if key - 1 == prev:
1005                         if not completing:
1006                             completing = True
1007                             f.write('-')
1008                     elif completing:
1009                         completing = False
1010                         f.write('%s %s' % (prev, key))
1011                     else:
1012                         f.write(' %s' % key)
1013                     prev = key
1014                 if completing:
1015                     f.write(str(prev) + '\n')
1016                 else:
1017                     f.write('\n')
1018         finally:
1019             f.close()
1020
1021     def pack(self):
1022         """Re-name messages to eliminate numbering gaps. Invalidates keys."""
1023         sequences = self.get_sequences()
1024         prev = 0
1025         changes = []
1026         for key in self.iterkeys():
1027             if key - 1 != prev:
1028                 changes.append((key, prev + 1))
1029                 f = open(os.path.join(self._path, str(key)), 'r+')
1030                 try:
1031                     if self._locked:
1032                         _lock_file(f)
1033                     try:
1034                         if hasattr(os, 'link'):
1035                             os.link(os.path.join(self._path, str(key)),
1036                                     os.path.join(self._path, str(prev + 1)))
1037                             if sys.platform == 'os2emx':
1038                                 # cannot unlink an open file on OS/2
1039                                 f.close()
1040                             os.unlink(os.path.join(self._path, str(key)))
1041                         else:
1042                             f.close()
1043                             os.rename(os.path.join(self._path, str(key)),
1044                                       os.path.join(self._path, str(prev + 1)))
1045                     finally:
1046                         if self._locked:
1047                             _unlock_file(f)
1048                 finally:
1049                     f.close()
1050             prev += 1
1051         self._next_key = prev + 1
1052         if len(changes) == 0:
1053             return
1054         for name, key_list in sequences.items():
1055             for old, new in changes:
1056                 if old in key_list:
1057                     key_list[key_list.index(old)] = new
1058         self.set_sequences(sequences)
1059
1060     def _dump_sequences(self, message, key):
1061         """Inspect a new MHMessage and update sequences appropriately."""
1062         pending_sequences = message.get_sequences()
1063         all_sequences = self.get_sequences()
1064         for name, key_list in all_sequences.iteritems():
1065             if name in pending_sequences:
1066                 key_list.append(key)
1067             elif key in key_list:
1068                 del key_list[key_list.index(key)]
1069         for sequence in pending_sequences:
1070             if sequence not in all_sequences:
1071                 all_sequences[sequence] = [key]
1072         self.set_sequences(all_sequences)
1073
1074
1075 class Babyl(_singlefileMailbox):
1076     """An Rmail-style Babyl mailbox."""
1077
1078     _special_labels = frozenset(('unseen', 'deleted', 'filed', 'answered',
1079                                  'forwarded', 'edited', 'resent'))
1080
1081     def __init__(self, path, factory=None, create=True):
1082         """Initialize a Babyl mailbox."""
1083         _singlefileMailbox.__init__(self, path, factory, create)
1084         self._labels = {}
1085
1086     def add(self, message):
1087         """Add message and return assigned key."""
1088         key = _singlefileMailbox.add(self, message)
1089         if isinstance(message, BabylMessage):
1090             self._labels[key] = message.get_labels()
1091         return key
1092
1093     def remove(self, key):
1094         """Remove the keyed message; raise KeyError if it doesn't exist."""
1095         _singlefileMailbox.remove(self, key)
1096         if key in self._labels:
1097             del self._labels[key]
1098
1099     def __setitem__(self, key, message):
1100         """Replace the keyed message; raise KeyError if it doesn't exist."""
1101         _singlefileMailbox.__setitem__(self, key, message)
1102         if isinstance(message, BabylMessage):
1103             self._labels[key] = message.get_labels()
1104
1105     def get_message(self, key):
1106         """Return a Message representation or raise a KeyError."""
1107         start, stop = self._lookup(key)
1108         self._file.seek(start)
1109         self._file.readline()   # Skip '1,' line specifying labels.
1110         original_headers = StringIO.StringIO()
1111         while True:
1112             line = self._file.readline()
1113             if line == '*** EOOH ***' + os.linesep or line == '':
1114                 break
1115             original_headers.write(line.replace(os.linesep, '\n'))
1116         visible_headers = StringIO.StringIO()
1117         while True:
1118             line = self._file.readline()
1119             if line == os.linesep or line == '':
1120                 break
1121             visible_headers.write(line.replace(os.linesep, '\n'))
1122         body = self._file.read(stop - self._file.tell()).replace(os.linesep,
1123                                                                  '\n')
1124         msg = BabylMessage(original_headers.getvalue() + body)
1125         msg.set_visible(visible_headers.getvalue())
1126         if key in self._labels:
1127             msg.set_labels(self._labels[key])
1128         return msg
1129
1130     def get_string(self, key):
1131         """Return a string representation or raise a KeyError."""
1132         start, stop = self._lookup(key)
1133         self._file.seek(start)
1134         self._file.readline()   # Skip '1,' line specifying labels.
1135         original_headers = StringIO.StringIO()
1136         while True:
1137             line = self._file.readline()
1138             if line == '*** EOOH ***' + os.linesep or line == '':
1139                 break
1140             original_headers.write(line.replace(os.linesep, '\n'))
1141         while True:
1142             line = self._file.readline()
1143             if line == os.linesep or line == '':
1144                 break
1145         return original_headers.getvalue() + \
1146                self._file.read(stop - self._file.tell()).replace(os.linesep,
1147                                                                  '\n')
1148
1149     def get_file(self, key):
1150         """Return a file-like representation or raise a KeyError."""
1151         return StringIO.StringIO(self.get_string(key).replace('\n',
1152                                                               os.linesep))
1153
1154     def get_labels(self):
1155         """Return a list of user-defined labels in the mailbox."""
1156         self._lookup()
1157         labels = set()
1158         for label_list in self._labels.values():
1159             labels.update(label_list)
1160         labels.difference_update(self._special_labels)
1161         return list(labels)
1162
1163     def _generate_toc(self):
1164         """Generate key-to-(start, stop) table of contents."""
1165         starts, stops = [], []
1166         self._file.seek(0)
1167         next_pos = 0
1168         label_lists = []
1169         while True:
1170             line_pos = next_pos
1171             line = self._file.readline()
1172             next_pos = self._file.tell()
1173             if line == '\037\014' + os.linesep:
1174                 if len(stops) < len(starts):
1175                     stops.append(line_pos - len(os.linesep))
1176                 starts.append(next_pos)
1177                 labels = [label.strip() for label
1178                                         in self._file.readline()[1:].split(',')
1179                                         if label.strip() != '']
1180                 label_lists.append(labels)
1181             elif line == '\037' or line == '\037' + os.linesep:
1182                 if len(stops) < len(starts):
1183                     stops.append(line_pos - len(os.linesep))
1184             elif line == '':
1185                 stops.append(line_pos - len(os.linesep))
1186                 break
1187         self._toc = dict(enumerate(zip(starts, stops)))
1188         self._labels = dict(enumerate(label_lists))
1189         self._next_key = len(self._toc)
1190
1191     def _pre_mailbox_hook(self, f):
1192         """Called before writing the mailbox to file f."""
1193         f.write('BABYL OPTIONS:%sVersion: 5%sLabels:%s%s\037' %
1194                 (os.linesep, os.linesep, ','.join(self.get_labels()),
1195                  os.linesep))
1196
1197     def _pre_message_hook(self, f):
1198         """Called before writing each message to file f."""
1199         f.write('\014' + os.linesep)
1200
1201     def _post_message_hook(self, f):
1202         """Called after writing each message to file f."""
1203         f.write(os.linesep + '\037')
1204
1205     def _install_message(self, message):
1206         """Write message contents and return (start, stop)."""
1207         start = self._file.tell()
1208         if isinstance(message, BabylMessage):
1209             special_labels = []
1210             labels = []
1211             for label in message.get_labels():
1212                 if label in self._special_labels:
1213                     special_labels.append(label)
1214                 else:
1215                     labels.append(label)
1216             self._file.write('1')
1217             for label in special_labels:
1218                 self._file.write(', ' + label)
1219             self._file.write(',,')
1220             for label in labels:
1221                 self._file.write(' ' + label + ',')
1222             self._file.write(os.linesep)
1223         else:
1224             self._file.write('1,,' + os.linesep)
1225         if isinstance(message, email.Message.Message):
1226             orig_buffer = StringIO.StringIO()
1227             orig_generator = email.Generator.Generator(orig_buffer, False, 0)
1228             orig_generator.flatten(message)
1229             orig_buffer.seek(0)
1230             while True:
1231                 line = orig_buffer.readline()
1232                 self._file.write(line.replace('\n', os.linesep))
1233                 if line == '\n' or line == '':
1234                     break
1235             self._file.write('*** EOOH ***' + os.linesep)
1236             if isinstance(message, BabylMessage):
1237                 vis_buffer = StringIO.StringIO()
1238                 vis_generator = email.Generator.Generator(vis_buffer, False, 0)
1239                 vis_generator.flatten(message.get_visible())
1240                 while True:
1241                     line = vis_buffer.readline()
1242                     self._file.write(line.replace('\n', os.linesep))
1243                     if line == '\n' or line == '':
1244                         break
1245             else:
1246                 orig_buffer.seek(0)
1247                 while True:
1248                     line = orig_buffer.readline()
1249                     self._file.write(line.replace('\n', os.linesep))
1250                     if line == '\n' or line == '':
1251                         break
1252             while True:
1253                 buffer = orig_buffer.read(4096) # Buffer size is arbitrary.
1254                 if buffer == '':
1255                     break
1256                 self._file.write(buffer.replace('\n', os.linesep))
1257         elif isinstance(message, str):
1258             body_start = message.find('\n\n') + 2
1259             if body_start - 2 != -1:
1260                 self._file.write(message[:body_start].replace('\n',
1261                                                               os.linesep))
1262                 self._file.write('*** EOOH ***' + os.linesep)
1263                 self._file.write(message[:body_start].replace('\n',
1264                                                               os.linesep))
1265                 self._file.write(message[body_start:].replace('\n',
1266                                                               os.linesep))
1267             else:
1268                 self._file.write('*** EOOH ***' + os.linesep + os.linesep)
1269                 self._file.write(message.replace('\n', os.linesep))
1270         elif hasattr(message, 'readline'):
1271             original_pos = message.tell()
1272             first_pass = True
1273             while True:
1274                 line = message.readline()
1275                 self._file.write(line.replace('\n', os.linesep))
1276                 if line == '\n' or line == '':
1277                     self._file.write('*** EOOH ***' + os.linesep)
1278                     if first_pass:
1279                         first_pass = False
1280                         message.seek(original_pos)
1281                     else:
1282                         break
1283             while True:
1284                 buffer = message.read(4096)     # Buffer size is arbitrary.
1285                 if buffer == '':
1286                     break
1287                 self._file.write(buffer.replace('\n', os.linesep))
1288         else:
1289             raise TypeError('Invalid message type: %s' % type(message))
1290         stop = self._file.tell()
1291         return (start, stop)
1292
1293
1294 class Message(email.Message.Message):
1295     """Message with mailbox-format-specific properties."""
1296
1297     def __init__(self, message=None):
1298         """Initialize a Message instance."""
1299         if isinstance(message, email.Message.Message):
1300             self._become_message(copy.deepcopy(message))
1301             if isinstance(message, Message):
1302                 message._explain_to(self)
1303         elif isinstance(message, str):
1304             self._become_message(email.message_from_string(message))
1305         elif hasattr(message, "read"):
1306             self._become_message(email.message_from_file(message))
1307         elif message is None:
1308             email.Message.Message.__init__(self)
1309         else:
1310             raise TypeError('Invalid message type: %s' % type(message))
1311
1312     def _become_message(self, message):
1313         """Assume the non-format-specific state of message."""
1314         for name in ('_headers', '_unixfrom', '_payload', '_charset',
1315                      'preamble', 'epilogue', 'defects', '_default_type'):
1316             self.__dict__[name] = message.__dict__[name]
1317
1318     def _explain_to(self, message):
1319         """Copy format-specific state to message insofar as possible."""
1320         if isinstance(message, Message):
1321             return  # There's nothing format-specific to explain.
1322         else:
1323             raise TypeError('Cannot convert to specified type')
1324
1325
1326 class MaildirMessage(Message):
1327     """Message with Maildir-specific properties."""
1328
1329     def __init__(self, message=None):
1330         """Initialize a MaildirMessage instance."""
1331         self._subdir = 'new'
1332         self._info = ''
1333         self._date = time.time()
1334         Message.__init__(self, message)
1335
1336     def get_subdir(self):
1337         """Return 'new' or 'cur'."""
1338         return self._subdir
1339
1340     def set_subdir(self, subdir):
1341         """Set subdir to 'new' or 'cur'."""
1342         if subdir == 'new' or subdir == 'cur':
1343             self._subdir = subdir
1344         else:
1345             raise ValueError("subdir must be 'new' or 'cur': %s" % subdir)
1346
1347     def get_flags(self):
1348         """Return as a string the flags that are set."""
1349         if self._info.startswith('2,'):
1350             return self._info[2:]
1351         else:
1352             return ''
1353
1354     def set_flags(self, flags):
1355         """Set the given flags and unset all others."""
1356         self._info = '2,' + ''.join(sorted(flags))
1357
1358     def add_flag(self, flag):
1359         """Set the given flag(s) without changing others."""
1360         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1361
1362     def remove_flag(self, flag):
1363         """Unset the given string flag(s) without changing others."""
1364         if self.get_flags() != '':
1365             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1366
1367     def get_date(self):
1368         """Return delivery date of message, in seconds since the epoch."""
1369         return self._date
1370
1371     def set_date(self, date):
1372         """Set delivery date of message, in seconds since the epoch."""
1373         try:
1374             self._date = float(date)
1375         except ValueError:
1376             raise TypeError("can't convert to float: %s" % date)
1377
1378     def get_info(self):
1379         """Get the message's "info" as a string."""
1380         return self._info
1381
1382     def set_info(self, info):
1383         """Set the message's "info" string."""
1384         if isinstance(info, str):
1385             self._info = info
1386         else:
1387             raise TypeError('info must be a string: %s' % type(info))
1388
1389     def _explain_to(self, message):
1390         """Copy Maildir-specific state to message insofar as possible."""
1391         if isinstance(message, MaildirMessage):
1392             message.set_flags(self.get_flags())
1393             message.set_subdir(self.get_subdir())
1394             message.set_date(self.get_date())
1395         elif isinstance(message, _mboxMMDFMessage):
1396             flags = set(self.get_flags())
1397             if 'S' in flags:
1398                 message.add_flag('R')
1399             if self.get_subdir() == 'cur':
1400                 message.add_flag('O')
1401             if 'T' in flags:
1402                 message.add_flag('D')
1403             if 'F' in flags:
1404                 message.add_flag('F')
1405             if 'R' in flags:
1406                 message.add_flag('A')
1407             message.set_from('MAILER-DAEMON', time.gmtime(self.get_date()))
1408         elif isinstance(message, MHMessage):
1409             flags = set(self.get_flags())
1410             if 'S' not in flags:
1411                 message.add_sequence('unseen')
1412             if 'R' in flags:
1413                 message.add_sequence('replied')
1414             if 'F' in flags:
1415                 message.add_sequence('flagged')
1416         elif isinstance(message, BabylMessage):
1417             flags = set(self.get_flags())
1418             if 'S' not in flags:
1419                 message.add_label('unseen')
1420             if 'T' in flags:
1421                 message.add_label('deleted')
1422             if 'R' in flags:
1423                 message.add_label('answered')
1424             if 'P' in flags:
1425                 message.add_label('forwarded')
1426         elif isinstance(message, Message):
1427             pass
1428         else:
1429             raise TypeError('Cannot convert to specified type: %s' %
1430                             type(message))
1431
1432
1433 class _mboxMMDFMessage(Message):
1434     """Message with mbox- or MMDF-specific properties."""
1435
1436     def __init__(self, message=None):
1437         """Initialize an mboxMMDFMessage instance."""
1438         self.set_from('MAILER-DAEMON', True)
1439         if isinstance(message, email.Message.Message):
1440             unixfrom = message.get_unixfrom()
1441             if unixfrom is not None and unixfrom.startswith('From '):
1442                 self.set_from(unixfrom[5:])
1443         Message.__init__(self, message)
1444
1445     def get_from(self):
1446         """Return contents of "From " line."""
1447         return self._from
1448
1449     def set_from(self, from_, time_=None):
1450         """Set "From " line, formatting and appending time_ if specified."""
1451         if time_ is not None:
1452             if time_ is True:
1453                 time_ = time.gmtime()
1454             from_ += ' ' + time.asctime(time_)
1455         self._from = from_
1456
1457     def get_flags(self):
1458         """Return as a string the flags that are set."""
1459         return self.get('Status', '') + self.get('X-Status', '')
1460
1461     def set_flags(self, flags):
1462         """Set the given flags and unset all others."""
1463         flags = set(flags)
1464         status_flags, xstatus_flags = '', ''
1465         for flag in ('R', 'O'):
1466             if flag in flags:
1467                 status_flags += flag
1468                 flags.remove(flag)
1469         for flag in ('D', 'F', 'A'):
1470             if flag in flags:
1471                 xstatus_flags += flag
1472                 flags.remove(flag)
1473         xstatus_flags += ''.join(sorted(flags))
1474         try:
1475             self.replace_header('Status', status_flags)
1476         except KeyError:
1477             self.add_header('Status', status_flags)
1478         try:
1479             self.replace_header('X-Status', xstatus_flags)
1480         except KeyError:
1481             self.add_header('X-Status', xstatus_flags)
1482
1483     def add_flag(self, flag):
1484         """Set the given flag(s) without changing others."""
1485         self.set_flags(''.join(set(self.get_flags()) | set(flag)))
1486
1487     def remove_flag(self, flag):
1488         """Unset the given string flag(s) without changing others."""
1489         if 'Status' in self or 'X-Status' in self:
1490             self.set_flags(''.join(set(self.get_flags()) - set(flag)))
1491
1492     def _explain_to(self, message):
1493         """Copy mbox- or MMDF-specific state to message insofar as possible."""
1494         if isinstance(message, MaildirMessage):
1495             flags = set(self.get_flags())
1496             if 'O' in flags:
1497                 message.set_subdir('cur')
1498             if 'F' in flags:
1499                 message.add_flag('F')
1500             if 'A' in flags:
1501                 message.add_flag('R')
1502             if 'R' in flags:
1503                 message.add_flag('S')
1504             if 'D' in flags:
1505                 message.add_flag('T')
1506             del message['status']
1507             del message['x-status']
1508             maybe_date = ' '.join(self.get_from().split()[-5:])
1509             try:
1510                 message.set_date(calendar.timegm(time.strptime(maybe_date,
1511                                                       '%a %b %d %H:%M:%S %Y')))
1512             except (ValueError, OverflowError):
1513                 pass
1514         elif isinstance(message, _mboxMMDFMessage):
1515             message.set_flags(self.get_flags())
1516             message.set_from(self.get_from())
1517         elif isinstance(message, MHMessage):
1518             flags = set(self.get_flags())
1519             if 'R' not in flags:
1520                 message.add_sequence('unseen')
1521             if 'A' in flags:
1522                 message.add_sequence('replied')
1523             if 'F' in flags:
1524                 message.add_sequence('flagged')
1525             del message['status']
1526             del message['x-status']
1527         elif isinstance(message, BabylMessage):
1528             flags = set(self.get_flags())
1529             if 'R' not in flags:
1530                 message.add_label('unseen')
1531             if 'D' in flags:
1532                 message.add_label('deleted')
1533             if 'A' in flags:
1534                 message.add_label('answered')
1535             del message['status']
1536             del message['x-status']
1537         elif isinstance(message, Message):
1538             pass
1539         else:
1540             raise TypeError('Cannot convert to specified type: %s' %
1541                             type(message))
1542
1543
1544 class mboxMessage(_mboxMMDFMessage):
1545     """Message with mbox-specific properties."""
1546
1547
1548 class MHMessage(Message):
1549     """Message with MH-specific properties."""
1550
1551     def __init__(self, message=None):
1552         """Initialize an MHMessage instance."""
1553         self._sequences = []
1554         Message.__init__(self, message)
1555
1556     def get_sequences(self):
1557         """Return a list of sequences that include the message."""
1558         return self._sequences[:]
1559
1560     def set_sequences(self, sequences):
1561         """Set the list of sequences that include the message."""
1562         self._sequences = list(sequences)
1563
1564     def add_sequence(self, sequence):
1565         """Add sequence to list of sequences including the message."""
1566         if isinstance(sequence, str):
1567             if not sequence in self._sequences:
1568                 self._sequences.append(sequence)
1569         else:
1570             raise TypeError('sequence must be a string: %s' % type(sequence))
1571
1572     def remove_sequence(self, sequence):
1573         """Remove sequence from the list of sequences including the message."""
1574         try:
1575             self._sequences.remove(sequence)
1576         except ValueError:
1577             pass
1578
1579     def _explain_to(self, message):
1580         """Copy MH-specific state to message insofar as possible."""
1581         if isinstance(message, MaildirMessage):
1582             sequences = set(self.get_sequences())
1583             if 'unseen' in sequences:
1584                 message.set_subdir('cur')
1585             else:
1586                 message.set_subdir('cur')
1587                 message.add_flag('S')
1588             if 'flagged' in sequences:
1589                 message.add_flag('F')
1590             if 'replied' in sequences:
1591                 message.add_flag('R')
1592         elif isinstance(message, _mboxMMDFMessage):
1593             sequences = set(self.get_sequences())
1594             if 'unseen' not in sequences:
1595                 message.add_flag('RO')
1596             else:
1597                 message.add_flag('O')
1598             if 'flagged' in sequences:
1599                 message.add_flag('F')
1600             if 'replied' in sequences:
1601                 message.add_flag('A')
1602         elif isinstance(message, MHMessage):
1603             for sequence in self.get_sequences():
1604                 message.add_sequence(sequence)
1605         elif isinstance(message, BabylMessage):
1606             sequences = set(self.get_sequences())
1607             if 'unseen' in sequences:
1608                 message.add_label('unseen')
1609             if 'replied' in sequences:
1610                 message.add_label('answered')
1611         elif isinstance(message, Message):
1612             pass
1613         else:
1614             raise TypeError('Cannot convert to specified type: %s' %
1615                             type(message))
1616
1617
1618 class BabylMessage(Message):
1619     """Message with Babyl-specific properties."""
1620
1621     def __init__(self, message=None):
1622         """Initialize an BabylMessage instance."""
1623         self._labels = []
1624         self._visible = Message()
1625         Message.__init__(self, message)
1626
1627     def get_labels(self):
1628         """Return a list of labels on the message."""
1629         return self._labels[:]
1630
1631     def set_labels(self, labels):
1632         """Set the list of labels on the message."""
1633         self._labels = list(labels)
1634
1635     def add_label(self, label):
1636         """Add label to list of labels on the message."""
1637         if isinstance(label, str):
1638             if label not in self._labels:
1639                 self._labels.append(label)
1640         else:
1641             raise TypeError('label must be a string: %s' % type(label))
1642
1643     def remove_label(self, label):
1644         """Remove label from the list of labels on the message."""
1645         try:
1646             self._labels.remove(label)
1647         except ValueError:
1648             pass
1649
1650     def get_visible(self):
1651         """Return a Message representation of visible headers."""
1652         return Message(self._visible)
1653
1654     def set_visible(self, visible):
1655         """Set the Message representation of visible headers."""
1656         self._visible = Message(visible)
1657
1658     def update_visible(self):
1659         """Update and/or sensibly generate a set of visible headers."""
1660         for header in self._visible.keys():
1661             if header in self:
1662                 self._visible.replace_header(header, self[header])
1663             else:
1664                 del self._visible[header]
1665         for header in ('Date', 'From', 'Reply-To', 'To', 'CC', 'Subject'):
1666             if header in self and header not in self._visible:
1667                 self._visible[header] = self[header]
1668
1669     def _explain_to(self, message):
1670         """Copy Babyl-specific state to message insofar as possible."""
1671         if isinstance(message, MaildirMessage):
1672             labels = set(self.get_labels())
1673             if 'unseen' in labels:
1674                 message.set_subdir('cur')
1675             else:
1676                 message.set_subdir('cur')
1677                 message.add_flag('S')
1678             if 'forwarded' in labels or 'resent' in labels:
1679                 message.add_flag('P')
1680             if 'answered' in labels:
1681                 message.add_flag('R')
1682             if 'deleted' in labels:
1683                 message.add_flag('T')
1684         elif isinstance(message, _mboxMMDFMessage):
1685             labels = set(self.get_labels())
1686             if 'unseen' not in labels:
1687                 message.add_flag('RO')
1688             else:
1689                 message.add_flag('O')
1690             if 'deleted' in labels:
1691                 message.add_flag('D')
1692             if 'answered' in labels:
1693                 message.add_flag('A')
1694         elif isinstance(message, MHMessage):
1695             labels = set(self.get_labels())
1696             if 'unseen' in labels:
1697                 message.add_sequence('unseen')
1698             if 'answered' in labels:
1699                 message.add_sequence('replied')
1700         elif isinstance(message, BabylMessage):
1701             message.set_visible(self.get_visible())
1702             for label in self.get_labels():
1703                 message.add_label(label)
1704         elif isinstance(message, Message):
1705             pass
1706         else:
1707             raise TypeError('Cannot convert to specified type: %s' %
1708                             type(message))
1709
1710
1711 class MMDFMessage(_mboxMMDFMessage):
1712     """Message with MMDF-specific properties."""
1713
1714
1715 class _ProxyFile:
1716     """A read-only wrapper of a file."""
1717
1718     def __init__(self, f, pos=None):
1719         """Initialize a _ProxyFile."""
1720         self._file = f
1721         if pos is None:
1722             self._pos = f.tell()
1723         else:
1724             self._pos = pos
1725
1726     def read(self, size=None):
1727         """Read bytes."""
1728         return self._read(size, self._file.read)
1729
1730     def readline(self, size=None):
1731         """Read a line."""
1732         return self._read(size, self._file.readline)
1733
1734     def readlines(self, sizehint=None):
1735         """Read multiple lines."""
1736         result = []
1737         for line in self:
1738             result.append(line)
1739             if sizehint is not None:
1740                 sizehint -= len(line)
1741                 if sizehint <= 0:
1742                     break
1743         return result
1744
1745     def __iter__(self):
1746         """Iterate over lines."""
1747         return iter(self.readline, "")
1748
1749     def tell(self):
1750         """Return the position."""
1751         return self._pos
1752
1753     def seek(self, offset, whence=0):
1754         """Change position."""
1755         if whence == 1:
1756             self._file.seek(self._pos)
1757         self._file.seek(offset, whence)
1758         self._pos = self._file.tell()
1759
1760     def close(self):
1761         """Close the file."""
1762         del self._file
1763
1764     def _read(self, size, read_method):
1765         """Read size bytes using read_method."""
1766         if size is None:
1767             size = -1
1768         self._file.seek(self._pos)
1769         result = read_method(size)
1770         self._pos = self._file.tell()
1771         return result
1772
1773
1774 class _PartialFile(_ProxyFile):
1775     """A read-only wrapper of part of a file."""
1776
1777     def __init__(self, f, start=None, stop=None):
1778         """Initialize a _PartialFile."""
1779         _ProxyFile.__init__(self, f, start)
1780         self._start = start
1781         self._stop = stop
1782
1783     def tell(self):
1784         """Return the position with respect to start."""
1785         return _ProxyFile.tell(self) - self._start
1786
1787     def seek(self, offset, whence=0):
1788         """Change position, possibly with respect to start or stop."""
1789         if whence == 0:
1790             self._pos = self._start
1791             whence = 1
1792         elif whence == 2:
1793             self._pos = self._stop
1794             whence = 1
1795         _ProxyFile.seek(self, offset, whence)
1796
1797     def _read(self, size, read_method):
1798         """Read size bytes using read_method, honoring start and stop."""
1799         remaining = self._stop - self._pos
1800         if remaining <= 0:
1801             return ''
1802         if size is None or size < 0 or size > remaining:
1803             size = remaining
1804         return _ProxyFile._read(self, size, read_method)
1805
1806
1807 def _lock_file(f, dotlock=True):
1808     """Lock file f using lockf and dot locking."""
1809     dotlock_done = False
1810     try:
1811         if fcntl:
1812             try:
1813                 fcntl.lockf(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
1814             except IOError, e:
1815                 if e.errno in (errno.EAGAIN, errno.EACCES):
1816                     raise ExternalClashError('lockf: lock unavailable: %s' %
1817                                              f.name)
1818                 else:
1819                     raise
1820         if dotlock:
1821             try:
1822                 pre_lock = _create_temporary(f.name + '.lock')
1823                 pre_lock.close()
1824             except IOError, e:
1825                 if e.errno == errno.EACCES:
1826                     return  # Without write access, just skip dotlocking.
1827                 else:
1828                     raise
1829             try:
1830                 if hasattr(os, 'link'):
1831                     os.link(pre_lock.name, f.name + '.lock')
1832                     dotlock_done = True
1833                     os.unlink(pre_lock.name)
1834                 else:
1835                     os.rename(pre_lock.name, f.name + '.lock')
1836                     dotlock_done = True
1837             except OSError, e:
1838                 if e.errno == errno.EEXIST or \
1839                   (os.name == 'os2' and e.errno == errno.EACCES):
1840                     os.remove(pre_lock.name)
1841                     raise ExternalClashError('dot lock unavailable: %s' %
1842                                              f.name)
1843                 else:
1844                     raise
1845     except:
1846         if fcntl:
1847             fcntl.lockf(f, fcntl.LOCK_UN)
1848         if dotlock_done:
1849             os.remove(f.name + '.lock')
1850         raise
1851
1852 def _unlock_file(f):
1853     """Unlock file f using lockf and dot locking."""
1854     if fcntl:
1855         fcntl.lockf(f, fcntl.LOCK_UN)
1856     if os.path.exists(f.name + '.lock'):
1857         os.remove(f.name + '.lock')
1858
1859 def _create_carefully(path):
1860     """Create a file if it doesn't exist and open for reading and writing."""
1861     fd = os.open(path, os.O_CREAT | os.O_EXCL | os.O_RDWR)
1862     try:
1863         return open(path, 'rb+')
1864     finally:
1865         os.close(fd)
1866
1867 def _create_temporary(path):
1868     """Create a temp file based on path and open for reading and writing."""
1869     return _create_carefully('%s.%s.%s.%s' % (path, int(time.time()),
1870                                               socket.gethostname(),
1871                                               os.getpid()))
1872
1873
1874 ## Start: classes from the original module (for backward compatibility).
1875
1876 # Note that the Maildir class, whose name is unchanged, itself offers a next()
1877 # method for backward compatibility.
1878
1879 class _Mailbox:
1880
1881     def __init__(self, fp, factory=rfc822.Message):
1882         self.fp = fp
1883         self.seekp = 0
1884         self.factory = factory
1885
1886     def __iter__(self):
1887         return iter(self.next, None)
1888
1889     def next(self):
1890         while 1:
1891             self.fp.seek(self.seekp)
1892             try:
1893                 self._search_start()
1894             except EOFError:
1895                 self.seekp = self.fp.tell()
1896                 return None
1897             start = self.fp.tell()
1898             self._search_end()
1899             self.seekp = stop = self.fp.tell()
1900             if start != stop:
1901                 break
1902         return self.factory(_PartialFile(self.fp, start, stop))
1903
1904 # Recommended to use PortableUnixMailbox instead!
1905 class UnixMailbox(_Mailbox):
1906
1907     def _search_start(self):
1908         while 1:
1909             pos = self.fp.tell()
1910             line = self.fp.readline()
1911             if not line:
1912                 raise EOFError
1913             if line[:5] == 'From ' and self._isrealfromline(line):
1914                 self.fp.seek(pos)
1915                 return
1916
1917     def _search_end(self):
1918         self.fp.readline()      # Throw away header line
1919         while 1:
1920             pos = self.fp.tell()
1921             line = self.fp.readline()
1922             if not line:
1923                 return
1924             if line[:5] == 'From ' and self._isrealfromline(line):
1925                 self.fp.seek(pos)
1926                 return
1927
1928     # An overridable mechanism to test for From-line-ness.  You can either
1929     # specify a different regular expression or define a whole new
1930     # _isrealfromline() method.  Note that this only gets called for lines
1931     # starting with the 5 characters "From ".
1932     #
1933     # BAW: According to
1934     #http://home.netscape.com/eng/mozilla/2.0/relnotes/demo/content-length.html
1935     # the only portable, reliable way to find message delimiters in a BSD (i.e
1936     # Unix mailbox) style folder is to search for "\n\nFrom .*\n", or at the
1937     # beginning of the file, "^From .*\n".  While _fromlinepattern below seems
1938     # like a good idea, in practice, there are too many variations for more
1939     # strict parsing of the line to be completely accurate.
1940     #
1941     # _strict_isrealfromline() is the old version which tries to do stricter
1942     # parsing of the From_ line.  _portable_isrealfromline() simply returns
1943     # true, since it's never called if the line doesn't already start with
1944     # "From ".
1945     #
1946     # This algorithm, and the way it interacts with _search_start() and
1947     # _search_end() may not be completely correct, because it doesn't check
1948     # that the two characters preceding "From " are \n\n or the beginning of
1949     # the file.  Fixing this would require a more extensive rewrite than is
1950     # necessary.  For convenience, we've added a PortableUnixMailbox class
1951     # which uses the more lenient _fromlinepattern regular expression.
1952
1953     _fromlinepattern = r"From \s*[^\s]+\s+\w\w\w\s+\w\w\w\s+\d?\d\s+" \
1954                        r"\d?\d:\d\d(:\d\d)?(\s+[^\s]+)?\s+\d\d\d\d\s*$"
1955     _regexp = None
1956
1957     def _strict_isrealfromline(self, line):
1958         if not self._regexp:
1959             import re
1960             self._regexp = re.compile(self._fromlinepattern)
1961         return self._regexp.match(line)
1962
1963     def _portable_isrealfromline(self, line):
1964         return True
1965
1966     _isrealfromline = _strict_isrealfromline
1967
1968
1969 class PortableUnixMailbox(UnixMailbox):
1970     _isrealfromline = UnixMailbox._portable_isrealfromline
1971
1972
1973 class MmdfMailbox(_Mailbox):
1974
1975     def _search_start(self):
1976         while 1:
1977             line = self.fp.readline()
1978             if not line:
1979                 raise EOFError
1980             if line[:5] == '\001\001\001\001\n':
1981                 return
1982
1983     def _search_end(self):
1984         while 1:
1985             pos = self.fp.tell()
1986             line = self.fp.readline()
1987             if not line:
1988                 return
1989             if line == '\001\001\001\001\n':
1990                 self.fp.seek(pos)
1991                 return
1992
1993
1994 class MHMailbox:
1995
1996     def __init__(self, dirname, factory=rfc822.Message):
1997         import re
1998         pat = re.compile('^[1-9][0-9]*$')
1999         self.dirname = dirname
2000         # the three following lines could be combined into:
2001         # list = map(long, filter(pat.match, os.listdir(self.dirname)))
2002         list = os.listdir(self.dirname)
2003         list = filter(pat.match, list)
2004         list = map(long, list)
2005         list.sort()
2006         # This only works in Python 1.6 or later;
2007         # before that str() added 'L':
2008         self.boxes = map(str, list)
2009         self.boxes.reverse()
2010         self.factory = factory
2011
2012     def __iter__(self):
2013         return iter(self.next, None)
2014
2015     def next(self):
2016         if not self.boxes:
2017             return None
2018         fn = self.boxes.pop()
2019         fp = open(os.path.join(self.dirname, fn))
2020         msg = self.factory(fp)
2021         try:
2022             msg._mh_msgno = fn
2023         except (AttributeError, TypeError):
2024             pass
2025         return msg
2026
2027
2028 class BabylMailbox(_Mailbox):
2029
2030     def _search_start(self):
2031         while 1:
2032             line = self.fp.readline()
2033             if not line:
2034                 raise EOFError
2035             if line == '*** EOOH ***\n':
2036                 return
2037
2038     def _search_end(self):
2039         while 1:
2040             pos = self.fp.tell()
2041             line = self.fp.readline()
2042             if not line:
2043                 return
2044             if line == '\037\014\n' or line == '\037':
2045                 self.fp.seek(pos)
2046                 return
2047
2048 ## End: classes from the original module (for backward compatibility).
2049
2050
2051 class Error(Exception):
2052     """Raised for module-specific errors."""
2053
2054 class NoSuchMailboxError(Error):
2055     """The specified mailbox does not exist and won't be created."""
2056
2057 class NotEmptyError(Error):
2058     """The specified mailbox is not empty and deletion was requested."""
2059
2060 class ExternalClashError(Error):
2061     """Another process caused an action to fail."""
2062
2063 class FormatError(Error):
2064     """A file appears to have an invalid format."""