Lib/bsddb/__init__.py

   1 #----------------------------------------------------------------------
   2 #  Copyright (c) 1999-2001, Digital Creations, Fredericksburg, VA, USA
   3 #  and Andrew Kuchling. All rights reserved.
   4 #
   5 #  Redistribution and use in source and binary forms, with or without
   6 #  modification, are permitted provided that the following conditions are
   7 #  met:
   8 #
   9 #    o Redistributions of source code must retain the above copyright
  10 #      notice, this list of conditions, and the disclaimer that follows.
  11 #
  12 #    o Redistributions in binary form must reproduce the above copyright
  13 #      notice, this list of conditions, and the following disclaimer in
  14 #      the documentation and/or other materials provided with the
  15 #      distribution.
  16 #
  17 #    o Neither the name of Digital Creations nor the names of its
  18 #      contributors may be used to endorse or promote products derived
  19 #      from this software without specific prior written permission.
  20 #
  21 #  THIS SOFTWARE IS PROVIDED BY DIGITAL CREATIONS AND CONTRIBUTORS *AS
  22 #  IS* AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
  23 #  TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
  24 #  PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL DIGITAL
  25 #  CREATIONS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26 #  INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27 #  BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
  28 #  OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
  29 #  ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
  30 #  TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
  31 #  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
  32 #  DAMAGE.
  33 #----------------------------------------------------------------------
  34
  35
  36 """Support for BerkeleyDB 3.2 through 4.2.
  37 """
  38
  39 try:
  40     if __name__ == 'bsddb3':
  41         # import _pybsddb binary as it should be the more recent version from
  42         # a standalone pybsddb addon package than the version included with
  43         # python as bsddb._bsddb.
  44         import _pybsddb
  45         _bsddb = _pybsddb
  46     else:
  47         import _bsddb
  48 except ImportError:
  49     # Remove ourselves from sys.modules
  50     import sys
  51     del sys.modules[__name__]
  52     raise
  53
  54 # bsddb3 calls it db, but provide _db for backwards compatibility
  55 db = _db = _bsddb
  56 __version__ = db.__version__
  57
  58 error = db.DBError  # So bsddb.error will mean something...
  59
  60 #----------------------------------------------------------------------
  61
  62 import sys, os
  63
  64 # for backwards compatibility with python versions older than 2.3, the
  65 # iterator interface is dynamically defined and added using a mixin
  66 # class.  old python can't tokenize it due to the yield keyword.
  67 if sys.version >= '2.3':
  68     exec """
  69 import UserDict
  70 from weakref import ref
  71 class _iter_mixin(UserDict.DictMixin):
  72     def _make_iter_cursor(self):
  73         cur = self.db.cursor()
  74         key = id(cur)
  75         self._cursor_refs[key] = ref(cur, self._gen_cref_cleaner(key))
  76         return cur
  77
  78     def _gen_cref_cleaner(self, key):
  79         # use generate the function for the weakref callback here
  80         # to ensure that we do not hold a strict reference to cur
  81         # in the callback.
  82         return lambda ref: self._cursor_refs.pop(key, None)
  83
  84     def __iter__(self):
  85         try:
  86             cur = self._make_iter_cursor()
  87
  88             # FIXME-20031102-greg: race condition.  cursor could
  89             # be closed by another thread before this call.
  90
  91             # since we're only returning keys, we call the cursor
  92             # methods with flags=0, dlen=0, dofs=0
  93             key = cur.first(0,0,0)[0]
  94             yield key
  95
  96             next = cur.next
  97             while 1:
  98                 try:
  99                     key = next(0,0,0)[0]
 100                     yield key
 101                 except _bsddb.DBCursorClosedError:
 102                     cur = self._make_iter_cursor()
 103                     # FIXME-20031101-greg: race condition.  cursor could
 104                     # be closed by another thread before this call.
 105                     cur.set(key,0,0,0)
 106                     next = cur.next
 107         except _bsddb.DBNotFoundError:
 108             return
 109         except _bsddb.DBCursorClosedError:
 110             # the database was modified during iteration.  abort.
 111             return
 112
 113     def iteritems(self):
 114         try:
 115             cur = self._make_iter_cursor()
 116
 117             # FIXME-20031102-greg: race condition.  cursor could
 118             # be closed by another thread before this call.
 119
 120             kv = cur.first()
 121             key = kv[0]
 122             yield kv
 123
 124             next = cur.next
 125             while 1:
 126                 try:
 127                     kv = next()
 128                     key = kv[0]
 129                     yield kv
 130                 except _bsddb.DBCursorClosedError:
 131                     cur = self._make_iter_cursor()
 132                     # FIXME-20031101-greg: race condition.  cursor could
 133                     # be closed by another thread before this call.
 134                     cur.set(key,0,0,0)
 135                     next = cur.next
 136         except _bsddb.DBNotFoundError:
 137             return
 138         except _bsddb.DBCursorClosedError:
 139             # the database was modified during iteration.  abort.
 140             return
 141 """
 142 else:
 143     class _iter_mixin: pass
 144
 145
 146 class _DBWithCursor(_iter_mixin):
 147     """
 148     A simple wrapper around DB that makes it look like the bsddbobject in
 149     the old module.  It uses a cursor as needed to provide DB traversal.
 150     """
 151     def __init__(self, db):
 152         self.db = db
 153         self.db.set_get_returns_none(0)
 154
 155         # FIXME-20031101-greg: I believe there is still the potential
 156         # for deadlocks in a multithreaded environment if someone
 157         # attempts to use the any of the cursor interfaces in one
 158         # thread while doing a put or delete in another thread.  The
 159         # reason is that _checkCursor and _closeCursors are not atomic
 160         # operations.  Doing our own locking around self.dbc,
 161         # self.saved_dbc_key and self._cursor_refs could prevent this.
 162         # TODO: A test case demonstrating the problem needs to be written.
 163
 164         # self.dbc is a DBCursor object used to implement the
 165         # first/next/previous/last/set_location methods.
 166         self.dbc = None
 167         self.saved_dbc_key = None
 168
 169         # a collection of all DBCursor objects currently allocated
 170         # by the _iter_mixin interface.
 171         self._cursor_refs = {}
 172
 173     def __del__(self):
 174         self.close()
 175
 176     def _checkCursor(self):
 177         if self.dbc is None:
 178             self.dbc = self.db.cursor()
 179             if self.saved_dbc_key is not None:
 180                 self.dbc.set(self.saved_dbc_key)
 181                 self.saved_dbc_key = None
 182
 183     # This method is needed for all non-cursor DB calls to avoid
 184     # BerkeleyDB deadlocks (due to being opened with DB_INIT_LOCK
 185     # and DB_THREAD to be thread safe) when intermixing database
 186     # operations that use the cursor internally with those that don't.
 187     def _closeCursors(self, save=1):
 188         if self.dbc:
 189             c = self.dbc
 190             self.dbc = None
 191             if save:
 192                 self.saved_dbc_key = c.current(0,0,0)[0]
 193             c.close()
 194             del c
 195         for cref in self._cursor_refs.values():
 196             c = cref()
 197             if c is not None:
 198                 c.close()
 199
 200     def _checkOpen(self):
 201         if self.db is None:
 202             raise error, "BSDDB object has already been closed"
 203
 204     def isOpen(self):
 205         return self.db is not None
 206
 207     def __len__(self):
 208         self._checkOpen()
 209         return len(self.db)
 210
 211     def __getitem__(self, key):
 212         self._checkOpen()
 213         return self.db[key]
 214
 215     def __setitem__(self, key, value):
 216         self._checkOpen()
 217         self._closeCursors()
 218         self.db[key] = value
 219
 220     def __delitem__(self, key):
 221         self._checkOpen()
 222         self._closeCursors()
 223         del self.db[key]
 224
 225     def close(self):
 226         self._closeCursors(save=0)
 227         if self.dbc is not None:
 228             self.dbc.close()
 229         v = 0
 230         if self.db is not None:
 231             v = self.db.close()
 232         self.dbc = None
 233         self.db = None
 234         return v
 235
 236     def keys(self):
 237         self._checkOpen()
 238         return self.db.keys()
 239
 240     def has_key(self, key):
 241         self._checkOpen()
 242         return self.db.has_key(key)
 243
 244     def set_location(self, key):
 245         self._checkOpen()
 246         self._checkCursor()
 247         return self.dbc.set_range(key)
 248
 249     def next(self):
 250         self._checkOpen()
 251         self._checkCursor()
 252         rv = self.dbc.next()
 253         return rv
 254
 255     def previous(self):
 256         self._checkOpen()
 257         self._checkCursor()
 258         rv = self.dbc.prev()
 259         return rv
 260
 261     def first(self):
 262         self._checkOpen()
 263         self._checkCursor()
 264         rv = self.dbc.first()
 265         return rv
 266
 267     def last(self):
 268         self._checkOpen()
 269         self._checkCursor()
 270         rv = self.dbc.last()
 271         return rv
 272
 273     def sync(self):
 274         self._checkOpen()
 275         return self.db.sync()
 276
 277
 278 #----------------------------------------------------------------------
 279 # Compatibility object factory functions
 280
 281 def hashopen(file, flag='c', mode=0666, pgsize=None, ffactor=None, nelem=None,
 282             cachesize=None, lorder=None, hflags=0):
 283
 284     flags = _checkflag(flag, file)
 285     e = _openDBEnv()
 286     d = db.DB(e)
 287     d.set_flags(hflags)
 288     if cachesize is not None: d.set_cachesize(0, cachesize)
 289     if pgsize is not None:    d.set_pagesize(pgsize)
 290     if lorder is not None:    d.set_lorder(lorder)
 291     if ffactor is not None:   d.set_h_ffactor(ffactor)
 292     if nelem is not None:     d.set_h_nelem(nelem)
 293     d.open(file, db.DB_HASH, flags, mode)
 294     return _DBWithCursor(d)
 295
 296 #----------------------------------------------------------------------
 297
 298 def btopen(file, flag='c', mode=0666,
 299             btflags=0, cachesize=None, maxkeypage=None, minkeypage=None,
 300             pgsize=None, lorder=None):
 301
 302     flags = _checkflag(flag, file)
 303     e = _openDBEnv()
 304     d = db.DB(e)
 305     if cachesize is not None: d.set_cachesize(0, cachesize)
 306     if pgsize is not None: d.set_pagesize(pgsize)
 307     if lorder is not None: d.set_lorder(lorder)
 308     d.set_flags(btflags)
 309     if minkeypage is not None: d.set_bt_minkey(minkeypage)
 310     if maxkeypage is not None: d.set_bt_maxkey(maxkeypage)
 311     d.open(file, db.DB_BTREE, flags, mode)
 312     return _DBWithCursor(d)
 313
 314 #----------------------------------------------------------------------
 315
 316
 317 def rnopen(file, flag='c', mode=0666,
 318             rnflags=0, cachesize=None, pgsize=None, lorder=None,
 319             rlen=None, delim=None, source=None, pad=None):
 320
 321     flags = _checkflag(flag, file)
 322     e = _openDBEnv()
 323     d = db.DB(e)
 324     if cachesize is not None: d.set_cachesize(0, cachesize)
 325     if pgsize is not None: d.set_pagesize(pgsize)
 326     if lorder is not None: d.set_lorder(lorder)
 327     d.set_flags(rnflags)
 328     if delim is not None: d.set_re_delim(delim)
 329     if rlen is not None: d.set_re_len(rlen)
 330     if source is not None: d.set_re_source(source)
 331     if pad is not None: d.set_re_pad(pad)
 332     d.open(file, db.DB_RECNO, flags, mode)
 333     return _DBWithCursor(d)
 334
 335 #----------------------------------------------------------------------
 336
 337 def _openDBEnv():
 338     e = db.DBEnv()
 339     e.open('.', db.DB_PRIVATE | db.DB_CREATE | db.DB_THREAD | db.DB_INIT_LOCK | db.DB_INIT_MPOOL)
 340     return e
 341
 342 def _checkflag(flag, file):
 343     if flag == 'r':
 344         flags = db.DB_RDONLY
 345     elif flag == 'rw':
 346         flags = 0
 347     elif flag == 'w':
 348         flags =  db.DB_CREATE
 349     elif flag == 'c':
 350         flags =  db.DB_CREATE
 351     elif flag == 'n':
 352         flags = db.DB_CREATE
 353         #flags = db.DB_CREATE | db.DB_TRUNCATE
 354         # we used db.DB_TRUNCATE flag for this before but BerkeleyDB
 355         # 4.2.52 changed to disallowed truncate with txn environments.
 356         if os.path.isfile(file):
 357             os.unlink(file)
 358     else:
 359         raise error, "flags should be one of 'r', 'w', 'c' or 'n'"
 360     return flags | db.DB_THREAD
 361
 362 #----------------------------------------------------------------------
 363
 364
 365 # This is a silly little hack that allows apps to continue to use the
 366 # DB_THREAD flag even on systems without threads without freaking out
 367 # BerkeleyDB.
 368 #
 369 # This assumes that if Python was built with thread support then
 370 # BerkeleyDB was too.
 371
 372 try:
 373     import thread
 374     del thread
 375 except ImportError:
 376     db.DB_THREAD = 0
 377
 378
 379 #----------------------------------------------------------------------