1 /* env.h : managing the BDB environment
3 * ====================================================================
4 * Copyright (c) 2000-2005 CollabNet. All rights reserved.
6 * This software is licensed as described in the file COPYING, which
7 * you should have received as part of this distribution. The terms
8 * are also available at http://subversion.tigris.org/license-1.html.
9 * If newer versions of this license are posted there, you may use a
10 * newer version instead, at your option.
12 * This software consists of voluntary contributions made by many
13 * individuals. For exact contribution history, see the revision
14 * history and logs, available at http://subversion.tigris.org/.
15 * ====================================================================
22 #include <apr_thread_mutex.h>
23 #include <apr_thread_proc.h>
27 #include <apr_strings.h>
31 #include "svn_pools.h"
33 #include "private/svn_atomic.h"
36 #include "bdb_compat.h"
40 /* A note about the BDB environment descriptor cache.
42 With the advent of DB_REGISTER in BDB-4.4, a process may only open
43 an environment handle once. This means that we must maintain a
44 cache of open environment handles, with reference counts. We
45 allocate each environment descriptor (a bdb_env_t) from its own
46 pool. The cache itself (and the cache pool) are shared between
47 threads, so all direct or indirect access to the pool is serialized
50 Because several threads can now use the same DB_ENV handle, we must
51 use the DB_THREAD flag when opening the environments, otherwise the
52 env handles (and all of libsvn_fs_base) won't be thread-safe.
54 If we use DB_THREAD, however, all of the code that reads data from
55 the database without a cursor must use either DB_DBT_MALLOC,
56 DB_DBT_REALLOC, or DB_DBT_USERMEM, as described in the BDB
59 (Oh, yes -- using DB_THREAD might not work on some systems. But
60 then, it's quite probable that threading is seriously broken on
61 those systems anyway, so we'll rely on APR_HAS_THREADS.)
65 /* The cache key for a Berkeley DB environment descriptor. This is a
66 combination of the device ID and INODE number of the Berkeley DB
69 XXX FIXME: Although the dev+inode combination is supposed do be
70 unique, apparently that's not always the case with some remote
71 filesystems. We /should/ be safe using this as a unique hash key,
72 because the database must be on a local filesystem. We can hope,
80 /* The cached Berkeley DB environment descriptor. */
83 /**************************************************************************/
86 /* Berkeley DB returns extended error info by callback before returning
87 an error code from the failing function. The callback baton type is a
88 string, not an arbitrary struct, so we prefix our struct with a valid
89 string, to avoid problems should BDB ever try to interpret our baton as
90 a string. Initializers of this structure must strcpy the value of
91 BDB_ERRPFX_STRING into this array. */
92 char errpfx_string
[sizeof(BDB_ERRPFX_STRING
)];
94 /* Extended error information. */
96 apr_threadkey_t
*error_info
; /* Points to a bdb_error_info_t. */
98 bdb_error_info_t error_info
;
101 /**************************************************************************/
102 /* BDB Environment Cache */
104 /* The Berkeley DB environment. */
107 /* The flags with which this environment was opened. Reopening the
108 environment with a different set of flags is not allowed. Trying
109 to change the state of the DB_PRIVATE flag is an especially bad
110 idea, so svn_fs_bdb__open() forbids any flag changes. */
113 /* The home path of this environment; a canonical SVN path encoded in
114 UTF-8 and allocated from this decriptor's pool. */
117 /* The home path of this environment, in the form expected by BDB. */
118 const char *path_bdb
;
120 /* The reference count for this environment handle; this is
121 essentially the difference between the number of calls to
122 svn_fs_bdb__open and svn_fs_bdb__close. */
125 /* If this flag is TRUE, someone has detected that the environment
126 descriptor is in a panicked state and should be removed from the
129 Note 1: Once this flag is set, it must not be cleared again.
131 Note 2: Unlike other fields in this structure, this field is not
132 protected by the cache mutex on threaded platforms, and
133 should only be accesses via the svn_atomic functions. */
134 volatile svn_atomic_t panic
;
136 /* The key for the environment descriptor cache. */
139 /* The handle of the open DB_CONFIG file.
141 We keep the DB_CONFIG file open in this process as long as the
142 environment handle itself is open. On Windows, this guarantees
143 that the cache key remains unique; here's what the Windows SDK
144 docs have to say about the file index (interpreted as the INODE
147 "This value is useful only while the file is open by at least
148 one process. If no processes have it open, the index may
149 change the next time the file is opened."
151 Now, we certainly don't want a unique key to change while it's
152 being used, do we... */
153 apr_file_t
*dbconfig_file
;
155 /* The pool associated with this environment descriptor.
157 Because the descriptor has a life of its own, the structure and
158 any data associated with it are allocated from their own global
166 /* Get the thread-specific error info from a bdb_env_t. */
167 static bdb_error_info_t
*
168 get_error_info(bdb_env_t
*bdb
)
171 apr_threadkey_private_get(&priv
, bdb
->error_info
);
174 priv
= calloc(1, sizeof(bdb_error_info_t
));
175 apr_threadkey_private_set(priv
, bdb
->error_info
);
180 #define get_error_info(bdb) (&(bdb)->error_info)
181 #endif /* APR_HAS_THREADS */
184 /* Convert a BDB error to a Subversion error. */
186 convert_bdb_error(bdb_env_t
*bdb
, int db_err
)
190 bdb_env_baton_t bdb_baton
;
191 bdb_baton
.env
= bdb
->env
;
193 bdb_baton
.error_info
= get_error_info(bdb
);
194 SVN_BDB_ERR(&bdb_baton
, db_err
);
200 /* Allocating an appropriate Berkeley DB environment object. */
202 /* BDB error callback. See bdb_error_info_t in env.h for more info.
203 Note: bdb_error_gatherer is a macro with BDB < 4.3, so be careful how
206 bdb_error_gatherer(const DB_ENV
*dbenv
, const char *baton
, const char *msg
)
208 bdb_error_info_t
*error_info
= get_error_info((bdb_env_t
*) baton
);
209 svn_error_t
*new_err
;
211 SVN_BDB_ERROR_GATHERER_IGNORE(dbenv
);
213 new_err
= svn_error_createf(SVN_NO_ERROR
, NULL
, "bdb: %s", msg
);
214 if (error_info
->pending_errors
)
215 svn_error_compose(error_info
->pending_errors
, new_err
);
217 error_info
->pending_errors
= new_err
;
219 if (error_info
->user_callback
)
220 error_info
->user_callback(NULL
, (char *)msg
); /* ### I hate this cast... */
224 /* Pool cleanup for the cached environment descriptor. */
226 cleanup_env(void *data
)
228 bdb_env_t
*bdb
= data
;
230 bdb
->dbconfig_file
= NULL
; /* will be closed during pool destruction */
232 apr_threadkey_private_delete(bdb
->error_info
);
233 #endif /* APR_HAS_THREADS */
235 /* If there are no references to this descriptor, free its memory here,
236 so that we don't leak it if create_env returns an error.
237 See bdb_close, which takes care of freeing this memory if the
238 environment is still open when the cache is destroyed. */
246 /* This cleanup is the fall back plan. If the thread exits and the
247 environment hasn't been closed it's responsible for cleanup of the
248 thread local error info variable, which would otherwise be leaked.
249 Normally it will not be called, because svn_fs_base__close will
250 set the thread's error info to NULL after cleaning it up. */
252 cleanup_error_info(void *baton
)
254 bdb_error_info_t
*error_info
= baton
;
257 svn_error_clear(error_info
->pending_errors
);
261 #endif /* APR_HAS_THREADS */
263 /* Create a Berkeley DB environment. */
265 create_env(bdb_env_t
**bdbp
, const char *path
, apr_pool_t
*pool
)
269 const char *path_bdb
;
270 char *tmp_path
, *tmp_path_bdb
;
271 apr_size_t path_size
, path_bdb_size
;
273 #if SVN_BDB_PATH_UTF8
274 path_bdb
= svn_path_local_style(path
, pool
);
276 SVN_ERR(svn_utf_cstring_from_utf8(&path_bdb
,
277 svn_path_local_style(path
, pool
),
281 /* Allocate the whole structure, including strings, from the heap,
282 because it must survive the cache pool cleanup. */
283 path_size
= strlen(path
) + 1;
284 path_bdb_size
= strlen(path_bdb
) + 1;
285 bdb
= calloc(1, sizeof(*bdb
) + path_size
+ path_bdb_size
);
287 /* We must initialize this now, as our callers may assume their bdb
288 pointer is valid when checking for errors. */
289 apr_pool_cleanup_register(pool
, bdb
, cleanup_env
, apr_pool_cleanup_null
);
290 apr_cpystrn(bdb
->errpfx_string
, BDB_ERRPFX_STRING
,
291 sizeof(bdb
->errpfx_string
));
292 bdb
->path
= tmp_path
= (char*)(bdb
+ 1);
293 bdb
->path_bdb
= tmp_path_bdb
= tmp_path
+ path_size
;
294 apr_cpystrn(tmp_path
, path
, path_size
);
295 apr_cpystrn(tmp_path_bdb
, path_bdb
, path_bdb_size
);
301 apr_status_t apr_err
= apr_threadkey_private_create(&bdb
->error_info
,
305 return svn_error_create(apr_err
, NULL
,
306 "Can't allocate thread-specific storage"
307 " for the Berkeley DB environment descriptor");
309 #endif /* APR_HAS_THREADS */
311 db_err
= db_env_create(&(bdb
->env
), 0);
314 bdb
->env
->set_errpfx(bdb
->env
, (char *) bdb
);
315 /* bdb_error_gatherer is in parens to stop macro expansion. */
316 bdb
->env
->set_errcall(bdb
->env
, (bdb_error_gatherer
));
318 /* Needed on Windows in case Subversion and Berkeley DB are using
319 different C runtime libraries */
320 db_err
= bdb
->env
->set_alloc(bdb
->env
, malloc
, realloc
, free
);
322 /* If we detect a deadlock, select a transaction to abort at
323 random from those participating in the deadlock. */
325 db_err
= bdb
->env
->set_lk_detect(bdb
->env
, DB_LOCK_RANDOM
);
327 return convert_bdb_error(bdb
, db_err
);
332 /* The environment descriptor cache. */
334 /* The global pool used for this cache. */
335 static apr_pool_t
*bdb_cache_pool
= NULL
;
337 /* The cache. The items are bdb_env_t structures. */
338 static apr_hash_t
*bdb_cache
= NULL
;
341 /* The mutex that protects bdb_cache. */
342 static apr_thread_mutex_t
*bdb_cache_lock
= NULL
;
344 /* Cleanup callback to NULL out the cache and its lock, so we don't try to
345 use them after the pool has been cleared during global shutdown. */
347 clear_cache(void *data
)
350 bdb_cache_lock
= NULL
;
353 #endif /* APR_HAS_THREADS */
355 static volatile svn_atomic_t bdb_cache_state
;
358 bdb_init_cb(apr_pool_t
*pool
)
361 apr_status_t apr_err
;
363 bdb_cache_pool
= svn_pool_create(pool
);
364 bdb_cache
= apr_hash_make(bdb_cache_pool
);
366 apr_err
= apr_thread_mutex_create(&bdb_cache_lock
,
367 APR_THREAD_MUTEX_DEFAULT
,
371 return svn_error_create(apr_err
, NULL
,
372 "Couldn't initialize the cache of"
373 " Berkeley DB environment descriptors");
375 apr_pool_cleanup_register(bdb_cache_pool
, NULL
, clear_cache
,
376 apr_pool_cleanup_null
);
377 #endif /* APR_HAS_THREADS */
383 svn_fs_bdb__init(apr_pool_t
* pool
)
385 SVN_ERR(svn_atomic__init_once(&bdb_cache_state
, bdb_init_cb
, pool
));
389 static APR_INLINE
void
390 acquire_cache_mutex(void)
394 apr_thread_mutex_lock(bdb_cache_lock
);
399 static APR_INLINE
void
400 release_cache_mutex(void)
404 apr_thread_mutex_unlock(bdb_cache_lock
);
409 /* Construct a cache key for the BDB environment at PATH in *KEYP.
410 if DBCONFIG_FILE is not NULL, return the opened file handle.
411 Allocate from POOL. */
413 bdb_cache_key(bdb_env_key_t
*keyp
, apr_file_t
**dbconfig_file
,
414 const char *path
, apr_pool_t
*pool
)
416 const char *dbcfg_file_name
= svn_path_join(path
, BDB_CONFIG_FILE
, pool
);
417 apr_file_t
*dbcfg_file
;
418 apr_status_t apr_err
;
421 SVN_ERR(svn_io_file_open(&dbcfg_file
, dbcfg_file_name
,
422 APR_READ
, APR_OS_DEFAULT
, pool
));
424 apr_err
= apr_file_info_get(&finfo
, APR_FINFO_DEV
| APR_FINFO_INODE
,
427 return svn_error_wrap_apr
428 (apr_err
, "Can't create BDB environment cache key");
430 /* Make sure that any padding in the key is always cleared, so that
431 the key's hash deterministic. */
432 memset(keyp
, 0, sizeof *keyp
);
433 keyp
->device
= finfo
.device
;
434 keyp
->inode
= finfo
.inode
;
437 *dbconfig_file
= dbcfg_file
;
439 apr_file_close(dbcfg_file
);
445 /* Find a BDB environment in the cache.
446 Return the environment's panic state in *PANICP.
448 Note: You MUST acquire the cache mutex before calling this function.
451 bdb_cache_get(const bdb_env_key_t
*keyp
, svn_boolean_t
*panicp
)
453 bdb_env_t
*bdb
= apr_hash_get(bdb_cache
, keyp
, sizeof *keyp
);
456 *panicp
= !!svn_atomic_read(&bdb
->panic
);
457 #if SVN_BDB_VERSION_AT_LEAST(4,2)
461 if (bdb
->env
->get_flags(bdb
->env
, &flags
)
462 || (flags
& DB_PANIC_ENVIRONMENT
))
464 /* Something is wrong with the environment. */
465 svn_atomic_set(&bdb
->panic
, TRUE
);
470 #endif /* at least bdb-4.2 */
481 /* Close and destroy a BDB environment descriptor. */
483 bdb_close(bdb_env_t
*bdb
)
485 svn_error_t
*err
= SVN_NO_ERROR
;
487 /* This bit is delcate; we must propagate the error from
488 DB_ENV->close to the caller, and always destroy the pool. */
489 int db_err
= bdb
->env
->close(bdb
->env
, 0);
491 /* If automatic database recovery is enabled, ignore DB_RUNRECOVERY
492 errors, since they're dealt with eventually by BDB itself. */
493 if (db_err
&& (!SVN_BDB_AUTO_RECOVER
|| db_err
!= DB_RUNRECOVERY
))
494 err
= convert_bdb_error(bdb
, db_err
);
496 /* Free the environment descriptor. The pool cleanup will do this unless
497 the cache has already been destroyed. */
499 svn_pool_destroy(bdb
->pool
);
507 svn_fs_bdb__close(bdb_env_baton_t
*bdb_baton
)
509 svn_error_t
*err
= SVN_NO_ERROR
;
510 bdb_env_t
*bdb
= bdb_baton
->bdb
;
512 assert(bdb_baton
->env
== bdb_baton
->bdb
->env
);
514 /* Neutralize bdb_baton's pool cleanup to prevent double-close. See
515 cleanup_env_baton(). */
516 bdb_baton
->bdb
= NULL
;
518 /* Note that we only bother with this cleanup if the pool is non-NULL, to
519 guard against potential races between this and the cleanup_env cleanup
520 callback. It's not clear if that can actually happen, but better safe
522 if (0 == --bdb_baton
->error_info
->refcount
&& bdb
->pool
)
524 svn_error_clear(bdb_baton
->error_info
->pending_errors
);
526 free(bdb_baton
->error_info
);
527 apr_threadkey_private_set(NULL
, bdb
->error_info
);
531 acquire_cache_mutex();
532 if (--bdb
->refcount
!= 0)
534 release_cache_mutex();
536 /* If the environment is panicked and automatic recovery is not
537 enabled, return an appropriate error. */
538 if (!SVN_BDB_AUTO_RECOVER
&& svn_atomic_read(&bdb
->panic
))
539 err
= svn_error_create(SVN_ERR_FS_BERKELEY_DB
, NULL
,
540 db_strerror(DB_RUNRECOVERY
));
544 /* If the bdb cache has been set to NULL that means we are
545 shutting down, and the pool that holds the bdb cache has
546 already been destroyed, so accessing it here would be a Bad
549 apr_hash_set(bdb_cache
, &bdb
->key
, sizeof bdb
->key
, NULL
);
550 err
= bdb_close(bdb
);
551 release_cache_mutex();
558 /* Open and initialize a BDB environment. */
560 bdb_open(bdb_env_t
*bdb
, u_int32_t flags
, int mode
)
565 SVN_ERR(convert_bdb_error
566 (bdb
, (bdb
->env
->open
)(bdb
->env
, bdb
->path_bdb
, flags
, mode
)));
568 #if SVN_BDB_AUTO_COMMIT
569 /* Assert the BDB_AUTO_COMMIT flag on the opened environment. This
570 will force all operations on the environment (and handles that
571 are opened within the environment) to be transactional. */
573 SVN_ERR(convert_bdb_error
574 (bdb
, bdb
->env
->set_flags(bdb
->env
, SVN_BDB_AUTO_COMMIT
, 1)));
577 SVN_ERR(bdb_cache_key(&bdb
->key
, &bdb
->dbconfig_file
,
578 bdb
->path
, bdb
->pool
));
584 /* Pool cleanup for the environment baton. */
586 cleanup_env_baton(void *data
)
588 bdb_env_baton_t
*bdb_baton
= data
;
591 svn_error_clear(svn_fs_bdb__close(bdb_baton
));
598 svn_fs_bdb__open(bdb_env_baton_t
**bdb_batonp
, const char *path
,
599 u_int32_t flags
, int mode
,
602 svn_error_t
*err
= SVN_NO_ERROR
;
607 acquire_cache_mutex();
609 /* We can safely discard the open DB_CONFIG file handle. If the
610 environment descriptor is in the cache, the key's immutability is
611 guaranteed. If it's not, we don't care if the key changes,
612 between here and the actual insertion of the newly-created
613 environment into the cache, because no other thread can touch the
614 cache in the meantime. */
615 err
= bdb_cache_key(&key
, NULL
, path
, pool
);
618 release_cache_mutex();
622 bdb
= bdb_cache_get(&key
, &panic
);
625 release_cache_mutex();
626 return svn_error_create(SVN_ERR_FS_BERKELEY_DB
, NULL
,
627 db_strerror(DB_RUNRECOVERY
));
630 /* Make sure that the environment's open flags haven't changed. */
631 if (bdb
&& bdb
->flags
!= flags
)
633 release_cache_mutex();
635 /* Handle changes to the DB_PRIVATE flag specially */
636 if ((flags
^ bdb
->flags
) & DB_PRIVATE
)
638 if (flags
& DB_PRIVATE
)
639 return svn_error_create(SVN_ERR_FS_BERKELEY_DB
, NULL
,
640 "Reopening a public Berkeley DB"
641 " environment with private attributes");
643 return svn_error_create(SVN_ERR_FS_BERKELEY_DB
, NULL
,
644 "Reopening a private Berkeley DB"
645 " environment with public attributes");
648 /* Otherwise return a generic "flags-mismatch" error. */
649 return svn_error_create(SVN_ERR_FS_BERKELEY_DB
, NULL
,
650 "Reopening a Berkeley DB environment"
651 " with different attributes");
656 err
= create_env(&bdb
, path
, svn_pool_create(bdb_cache_pool
));
659 err
= bdb_open(bdb
, flags
, mode
);
662 apr_hash_set(bdb_cache
, &bdb
->key
, sizeof bdb
->key
, bdb
);
668 /* Clean up, and we can't do anything about returned errors. */
669 svn_error_clear(bdb_close(bdb
));
680 *bdb_batonp
= apr_palloc(pool
, sizeof **bdb_batonp
);
681 (*bdb_batonp
)->env
= bdb
->env
;
682 (*bdb_batonp
)->bdb
= bdb
;
683 (*bdb_batonp
)->error_info
= get_error_info(bdb
);
684 ++(*bdb_batonp
)->error_info
->refcount
;
685 apr_pool_cleanup_register(pool
, *bdb_batonp
, cleanup_env_baton
,
686 apr_pool_cleanup_null
);
689 release_cache_mutex();
695 svn_fs_bdb__get_panic(bdb_env_baton_t
*bdb_baton
)
697 /* An invalid baton is equivalent to a panicked environment; in both
698 cases, database cleanups should be skipped. */
702 assert(bdb_baton
->env
== bdb_baton
->bdb
->env
);
703 return !!svn_atomic_read(&bdb_baton
->bdb
->panic
);
707 svn_fs_bdb__set_panic(bdb_env_baton_t
*bdb_baton
)
712 assert(bdb_baton
->env
== bdb_baton
->bdb
->env
);
713 svn_atomic_set(&bdb_baton
->bdb
->panic
, TRUE
);
717 /* This function doesn't actually open the environment, so it doesn't
718 have to look in the cache. Callers are supposed to own an
719 exclusive lock on the filesystem anyway. */
721 svn_fs_bdb__remove(const char *path
, apr_pool_t
*pool
)
725 SVN_ERR(create_env(&bdb
, path
, pool
));
726 return convert_bdb_error
727 (bdb
, bdb
->env
->remove(bdb
->env
, bdb
->path_bdb
, DB_FORCE
));