Mark many merge tests as skip-against-old-server.
[svn.git] / subversion / libsvn_fs_base / bdb / env.c
blob50ec6cf3abf86d8e79439491f90729de476163a1
1 /* env.h : managing the BDB environment
3 * ====================================================================
4 * Copyright (c) 2000-2005 CollabNet. All rights reserved.
6 * This software is licensed as described in the file COPYING, which
7 * you should have received as part of this distribution. The terms
8 * are also available at http://subversion.tigris.org/license-1.html.
9 * If newer versions of this license are posted there, you may use a
10 * newer version instead, at your option.
12 * This software consists of voluntary contributions made by many
13 * individuals. For exact contribution history, see the revision
14 * history and logs, available at http://subversion.tigris.org/.
15 * ====================================================================
18 #include <assert.h>
20 #include <apr.h>
21 #if APR_HAS_THREADS
22 #include <apr_thread_mutex.h>
23 #include <apr_thread_proc.h>
24 #include <apr_time.h>
25 #endif
27 #include <apr_strings.h>
28 #include <apr_hash.h>
30 #include "svn_path.h"
31 #include "svn_pools.h"
32 #include "svn_utf.h"
33 #include "private/svn_atomic.h"
35 #include "bdb-err.h"
36 #include "bdb_compat.h"
38 #include "env.h"
40 /* A note about the BDB environment descriptor cache.
42 With the advent of DB_REGISTER in BDB-4.4, a process may only open
43 an environment handle once. This means that we must maintain a
44 cache of open environment handles, with reference counts. We
45 allocate each environment descriptor (a bdb_env_t) from its own
46 pool. The cache itself (and the cache pool) are shared between
47 threads, so all direct or indirect access to the pool is serialized
48 with a global mutex.
50 Because several threads can now use the same DB_ENV handle, we must
51 use the DB_THREAD flag when opening the environments, otherwise the
52 env handles (and all of libsvn_fs_base) won't be thread-safe.
54 If we use DB_THREAD, however, all of the code that reads data from
55 the database without a cursor must use either DB_DBT_MALLOC,
56 DB_DBT_REALLOC, or DB_DBT_USERMEM, as described in the BDB
57 documentation.
59 (Oh, yes -- using DB_THREAD might not work on some systems. But
60 then, it's quite probable that threading is seriously broken on
61 those systems anyway, so we'll rely on APR_HAS_THREADS.)
65 /* The cache key for a Berkeley DB environment descriptor. This is a
66 combination of the device ID and INODE number of the Berkeley DB
67 config file.
69 XXX FIXME: Although the dev+inode combination is supposed do be
70 unique, apparently that's not always the case with some remote
71 filesystems. We /should/ be safe using this as a unique hash key,
72 because the database must be on a local filesystem. We can hope,
73 anyway. */
74 typedef struct
76 apr_dev_t device;
77 apr_ino_t inode;
78 } bdb_env_key_t;
80 /* The cached Berkeley DB environment descriptor. */
81 struct bdb_env_t
83 /**************************************************************************/
84 /* Error Reporting */
86 /* Berkeley DB returns extended error info by callback before returning
87 an error code from the failing function. The callback baton type is a
88 string, not an arbitrary struct, so we prefix our struct with a valid
89 string, to avoid problems should BDB ever try to interpret our baton as
90 a string. Initializers of this structure must strcpy the value of
91 BDB_ERRPFX_STRING into this array. */
92 char errpfx_string[sizeof(BDB_ERRPFX_STRING)];
94 /* Extended error information. */
95 #if APR_HAS_THREADS
96 apr_threadkey_t *error_info; /* Points to a bdb_error_info_t. */
97 #else
98 bdb_error_info_t error_info;
99 #endif
101 /**************************************************************************/
102 /* BDB Environment Cache */
104 /* The Berkeley DB environment. */
105 DB_ENV *env;
107 /* The flags with which this environment was opened. Reopening the
108 environment with a different set of flags is not allowed. Trying
109 to change the state of the DB_PRIVATE flag is an especially bad
110 idea, so svn_fs_bdb__open() forbids any flag changes. */
111 u_int32_t flags;
113 /* The home path of this environment; a canonical SVN path encoded in
114 UTF-8 and allocated from this decriptor's pool. */
115 const char *path;
117 /* The home path of this environment, in the form expected by BDB. */
118 const char *path_bdb;
120 /* The reference count for this environment handle; this is
121 essentially the difference between the number of calls to
122 svn_fs_bdb__open and svn_fs_bdb__close. */
123 unsigned refcount;
125 /* If this flag is TRUE, someone has detected that the environment
126 descriptor is in a panicked state and should be removed from the
127 cache.
129 Note 1: Once this flag is set, it must not be cleared again.
131 Note 2: Unlike other fields in this structure, this field is not
132 protected by the cache mutex on threaded platforms, and
133 should only be accesses via the svn_atomic functions. */
134 volatile svn_atomic_t panic;
136 /* The key for the environment descriptor cache. */
137 bdb_env_key_t key;
139 /* The handle of the open DB_CONFIG file.
141 We keep the DB_CONFIG file open in this process as long as the
142 environment handle itself is open. On Windows, this guarantees
143 that the cache key remains unique; here's what the Windows SDK
144 docs have to say about the file index (interpreted as the INODE
145 number by APR):
147 "This value is useful only while the file is open by at least
148 one process. If no processes have it open, the index may
149 change the next time the file is opened."
151 Now, we certainly don't want a unique key to change while it's
152 being used, do we... */
153 apr_file_t *dbconfig_file;
155 /* The pool associated with this environment descriptor.
157 Because the descriptor has a life of its own, the structure and
158 any data associated with it are allocated from their own global
159 pool. */
160 apr_pool_t *pool;
165 #if APR_HAS_THREADS
166 /* Get the thread-specific error info from a bdb_env_t. */
167 static bdb_error_info_t *
168 get_error_info(bdb_env_t *bdb)
170 void *priv;
171 apr_threadkey_private_get(&priv, bdb->error_info);
172 if (!priv)
174 priv = calloc(1, sizeof(bdb_error_info_t));
175 apr_threadkey_private_set(priv, bdb->error_info);
177 return priv;
179 #else
180 #define get_error_info(bdb) (&(bdb)->error_info)
181 #endif /* APR_HAS_THREADS */
184 /* Convert a BDB error to a Subversion error. */
185 static svn_error_t *
186 convert_bdb_error(bdb_env_t *bdb, int db_err)
188 if (db_err)
190 bdb_env_baton_t bdb_baton;
191 bdb_baton.env = bdb->env;
192 bdb_baton.bdb = bdb;
193 bdb_baton.error_info = get_error_info(bdb);
194 SVN_BDB_ERR(&bdb_baton, db_err);
196 return SVN_NO_ERROR;
200 /* Allocating an appropriate Berkeley DB environment object. */
202 /* BDB error callback. See bdb_error_info_t in env.h for more info.
203 Note: bdb_error_gatherer is a macro with BDB < 4.3, so be careful how
204 you use it! */
205 static void
206 bdb_error_gatherer(const DB_ENV *dbenv, const char *baton, const char *msg)
208 bdb_error_info_t *error_info = get_error_info((bdb_env_t *) baton);
209 svn_error_t *new_err;
211 SVN_BDB_ERROR_GATHERER_IGNORE(dbenv);
213 new_err = svn_error_createf(SVN_NO_ERROR, NULL, "bdb: %s", msg);
214 if (error_info->pending_errors)
215 svn_error_compose(error_info->pending_errors, new_err);
216 else
217 error_info->pending_errors = new_err;
219 if (error_info->user_callback)
220 error_info->user_callback(NULL, (char *)msg); /* ### I hate this cast... */
224 /* Pool cleanup for the cached environment descriptor. */
225 static apr_status_t
226 cleanup_env(void *data)
228 bdb_env_t *bdb = data;
229 bdb->pool = NULL;
230 bdb->dbconfig_file = NULL; /* will be closed during pool destruction */
231 #if APR_HAS_THREADS
232 apr_threadkey_private_delete(bdb->error_info);
233 #endif /* APR_HAS_THREADS */
235 /* If there are no references to this descriptor, free its memory here,
236 so that we don't leak it if create_env returns an error.
237 See bdb_close, which takes care of freeing this memory if the
238 environment is still open when the cache is destroyed. */
239 if (!bdb->refcount)
240 free(data);
242 return APR_SUCCESS;
245 #if APR_HAS_THREADS
246 /* This cleanup is the fall back plan. If the thread exits and the
247 environment hasn't been closed it's responsible for cleanup of the
248 thread local error info variable, which would otherwise be leaked.
249 Normally it will not be called, because svn_fs_base__close will
250 set the thread's error info to NULL after cleaning it up. */
251 static void
252 cleanup_error_info(void *baton)
254 bdb_error_info_t *error_info = baton;
256 if (error_info)
257 svn_error_clear(error_info->pending_errors);
259 free(error_info);
261 #endif /* APR_HAS_THREADS */
263 /* Create a Berkeley DB environment. */
264 static svn_error_t *
265 create_env(bdb_env_t **bdbp, const char *path, apr_pool_t *pool)
267 int db_err;
268 bdb_env_t *bdb;
269 const char *path_bdb;
270 char *tmp_path, *tmp_path_bdb;
271 apr_size_t path_size, path_bdb_size;
273 #if SVN_BDB_PATH_UTF8
274 path_bdb = svn_path_local_style(path, pool);
275 #else
276 SVN_ERR(svn_utf_cstring_from_utf8(&path_bdb,
277 svn_path_local_style(path, pool),
278 pool));
279 #endif
281 /* Allocate the whole structure, including strings, from the heap,
282 because it must survive the cache pool cleanup. */
283 path_size = strlen(path) + 1;
284 path_bdb_size = strlen(path_bdb) + 1;
285 bdb = calloc(1, sizeof(*bdb) + path_size + path_bdb_size);
287 /* We must initialize this now, as our callers may assume their bdb
288 pointer is valid when checking for errors. */
289 apr_pool_cleanup_register(pool, bdb, cleanup_env, apr_pool_cleanup_null);
290 apr_cpystrn(bdb->errpfx_string, BDB_ERRPFX_STRING,
291 sizeof(bdb->errpfx_string));
292 bdb->path = tmp_path = (char*)(bdb + 1);
293 bdb->path_bdb = tmp_path_bdb = tmp_path + path_size;
294 apr_cpystrn(tmp_path, path, path_size);
295 apr_cpystrn(tmp_path_bdb, path_bdb, path_bdb_size);
296 bdb->pool = pool;
297 *bdbp = bdb;
299 #if APR_HAS_THREADS
301 apr_status_t apr_err = apr_threadkey_private_create(&bdb->error_info,
302 cleanup_error_info,
303 pool);
304 if (apr_err)
305 return svn_error_create(apr_err, NULL,
306 "Can't allocate thread-specific storage"
307 " for the Berkeley DB environment descriptor");
309 #endif /* APR_HAS_THREADS */
311 db_err = db_env_create(&(bdb->env), 0);
312 if (!db_err)
314 bdb->env->set_errpfx(bdb->env, (char *) bdb);
315 /* bdb_error_gatherer is in parens to stop macro expansion. */
316 bdb->env->set_errcall(bdb->env, (bdb_error_gatherer));
318 /* Needed on Windows in case Subversion and Berkeley DB are using
319 different C runtime libraries */
320 db_err = bdb->env->set_alloc(bdb->env, malloc, realloc, free);
322 /* If we detect a deadlock, select a transaction to abort at
323 random from those participating in the deadlock. */
324 if (!db_err)
325 db_err = bdb->env->set_lk_detect(bdb->env, DB_LOCK_RANDOM);
327 return convert_bdb_error(bdb, db_err);
332 /* The environment descriptor cache. */
334 /* The global pool used for this cache. */
335 static apr_pool_t *bdb_cache_pool = NULL;
337 /* The cache. The items are bdb_env_t structures. */
338 static apr_hash_t *bdb_cache = NULL;
340 #if APR_HAS_THREADS
341 /* The mutex that protects bdb_cache. */
342 static apr_thread_mutex_t *bdb_cache_lock = NULL;
344 /* Cleanup callback to NULL out the cache and its lock, so we don't try to
345 use them after the pool has been cleared during global shutdown. */
346 static apr_status_t
347 clear_cache(void *data)
349 bdb_cache = NULL;
350 bdb_cache_lock = NULL;
351 return APR_SUCCESS;
353 #endif /* APR_HAS_THREADS */
355 static volatile svn_atomic_t bdb_cache_state;
357 static svn_error_t *
358 bdb_init_cb(apr_pool_t *pool)
360 #if APR_HAS_THREADS
361 apr_status_t apr_err;
362 #endif
363 bdb_cache_pool = svn_pool_create(pool);
364 bdb_cache = apr_hash_make(bdb_cache_pool);
365 #if APR_HAS_THREADS
366 apr_err = apr_thread_mutex_create(&bdb_cache_lock,
367 APR_THREAD_MUTEX_DEFAULT,
368 bdb_cache_pool);
369 if (apr_err)
371 return svn_error_create(apr_err, NULL,
372 "Couldn't initialize the cache of"
373 " Berkeley DB environment descriptors");
375 apr_pool_cleanup_register(bdb_cache_pool, NULL, clear_cache,
376 apr_pool_cleanup_null);
377 #endif /* APR_HAS_THREADS */
379 return SVN_NO_ERROR;
382 svn_error_t *
383 svn_fs_bdb__init(apr_pool_t* pool)
385 SVN_ERR(svn_atomic__init_once(&bdb_cache_state, bdb_init_cb, pool));
386 return SVN_NO_ERROR;
389 static APR_INLINE void
390 acquire_cache_mutex(void)
392 #if APR_HAS_THREADS
393 if (bdb_cache_lock)
394 apr_thread_mutex_lock(bdb_cache_lock);
395 #endif
399 static APR_INLINE void
400 release_cache_mutex(void)
402 #if APR_HAS_THREADS
403 if (bdb_cache_lock)
404 apr_thread_mutex_unlock(bdb_cache_lock);
405 #endif
409 /* Construct a cache key for the BDB environment at PATH in *KEYP.
410 if DBCONFIG_FILE is not NULL, return the opened file handle.
411 Allocate from POOL. */
412 static svn_error_t *
413 bdb_cache_key(bdb_env_key_t *keyp, apr_file_t **dbconfig_file,
414 const char *path, apr_pool_t *pool)
416 const char *dbcfg_file_name = svn_path_join(path, BDB_CONFIG_FILE, pool);
417 apr_file_t *dbcfg_file;
418 apr_status_t apr_err;
419 apr_finfo_t finfo;
421 SVN_ERR(svn_io_file_open(&dbcfg_file, dbcfg_file_name,
422 APR_READ, APR_OS_DEFAULT, pool));
424 apr_err = apr_file_info_get(&finfo, APR_FINFO_DEV | APR_FINFO_INODE,
425 dbcfg_file);
426 if (apr_err)
427 return svn_error_wrap_apr
428 (apr_err, "Can't create BDB environment cache key");
430 /* Make sure that any padding in the key is always cleared, so that
431 the key's hash deterministic. */
432 memset(keyp, 0, sizeof *keyp);
433 keyp->device = finfo.device;
434 keyp->inode = finfo.inode;
436 if (dbconfig_file)
437 *dbconfig_file = dbcfg_file;
438 else
439 apr_file_close(dbcfg_file);
441 return SVN_NO_ERROR;
445 /* Find a BDB environment in the cache.
446 Return the environment's panic state in *PANICP.
448 Note: You MUST acquire the cache mutex before calling this function.
450 static bdb_env_t *
451 bdb_cache_get(const bdb_env_key_t *keyp, svn_boolean_t *panicp)
453 bdb_env_t *bdb = apr_hash_get(bdb_cache, keyp, sizeof *keyp);
454 if (bdb && bdb->env)
456 *panicp = !!svn_atomic_read(&bdb->panic);
457 #if SVN_BDB_VERSION_AT_LEAST(4,2)
458 if (!*panicp)
460 u_int32_t flags;
461 if (bdb->env->get_flags(bdb->env, &flags)
462 || (flags & DB_PANIC_ENVIRONMENT))
464 /* Something is wrong with the environment. */
465 svn_atomic_set(&bdb->panic, TRUE);
466 *panicp = TRUE;
467 bdb = NULL;
470 #endif /* at least bdb-4.2 */
472 else
474 *panicp = FALSE;
476 return bdb;
481 /* Close and destroy a BDB environment descriptor. */
482 static svn_error_t *
483 bdb_close(bdb_env_t *bdb)
485 svn_error_t *err = SVN_NO_ERROR;
487 /* This bit is delcate; we must propagate the error from
488 DB_ENV->close to the caller, and always destroy the pool. */
489 int db_err = bdb->env->close(bdb->env, 0);
491 /* If automatic database recovery is enabled, ignore DB_RUNRECOVERY
492 errors, since they're dealt with eventually by BDB itself. */
493 if (db_err && (!SVN_BDB_AUTO_RECOVER || db_err != DB_RUNRECOVERY))
494 err = convert_bdb_error(bdb, db_err);
496 /* Free the environment descriptor. The pool cleanup will do this unless
497 the cache has already been destroyed. */
498 if (bdb->pool)
499 svn_pool_destroy(bdb->pool);
500 else
501 free(bdb);
502 return err;
506 svn_error_t *
507 svn_fs_bdb__close(bdb_env_baton_t *bdb_baton)
509 svn_error_t *err = SVN_NO_ERROR;
510 bdb_env_t *bdb = bdb_baton->bdb;
512 assert(bdb_baton->env == bdb_baton->bdb->env);
514 /* Neutralize bdb_baton's pool cleanup to prevent double-close. See
515 cleanup_env_baton(). */
516 bdb_baton->bdb = NULL;
518 /* Note that we only bother with this cleanup if the pool is non-NULL, to
519 guard against potential races between this and the cleanup_env cleanup
520 callback. It's not clear if that can actually happen, but better safe
521 than sorry. */
522 if (0 == --bdb_baton->error_info->refcount && bdb->pool)
524 svn_error_clear(bdb_baton->error_info->pending_errors);
525 #if APR_HAS_THREADS
526 free(bdb_baton->error_info);
527 apr_threadkey_private_set(NULL, bdb->error_info);
528 #endif
531 acquire_cache_mutex();
532 if (--bdb->refcount != 0)
534 release_cache_mutex();
536 /* If the environment is panicked and automatic recovery is not
537 enabled, return an appropriate error. */
538 if (!SVN_BDB_AUTO_RECOVER && svn_atomic_read(&bdb->panic))
539 err = svn_error_create(SVN_ERR_FS_BERKELEY_DB, NULL,
540 db_strerror(DB_RUNRECOVERY));
542 else
544 /* If the bdb cache has been set to NULL that means we are
545 shutting down, and the pool that holds the bdb cache has
546 already been destroyed, so accessing it here would be a Bad
547 Thing (tm) */
548 if (bdb_cache)
549 apr_hash_set(bdb_cache, &bdb->key, sizeof bdb->key, NULL);
550 err = bdb_close(bdb);
551 release_cache_mutex();
553 return err;
558 /* Open and initialize a BDB environment. */
559 static svn_error_t *
560 bdb_open(bdb_env_t *bdb, u_int32_t flags, int mode)
562 #if APR_HAS_THREADS
563 flags |= DB_THREAD;
564 #endif
565 SVN_ERR(convert_bdb_error
566 (bdb, (bdb->env->open)(bdb->env, bdb->path_bdb, flags, mode)));
568 #if SVN_BDB_AUTO_COMMIT
569 /* Assert the BDB_AUTO_COMMIT flag on the opened environment. This
570 will force all operations on the environment (and handles that
571 are opened within the environment) to be transactional. */
573 SVN_ERR(convert_bdb_error
574 (bdb, bdb->env->set_flags(bdb->env, SVN_BDB_AUTO_COMMIT, 1)));
575 #endif
577 SVN_ERR(bdb_cache_key(&bdb->key, &bdb->dbconfig_file,
578 bdb->path, bdb->pool));
580 return SVN_NO_ERROR;
584 /* Pool cleanup for the environment baton. */
585 static apr_status_t
586 cleanup_env_baton(void *data)
588 bdb_env_baton_t *bdb_baton = data;
590 if (bdb_baton->bdb)
591 svn_error_clear(svn_fs_bdb__close(bdb_baton));
593 return APR_SUCCESS;
597 svn_error_t *
598 svn_fs_bdb__open(bdb_env_baton_t **bdb_batonp, const char *path,
599 u_int32_t flags, int mode,
600 apr_pool_t *pool)
602 svn_error_t *err = SVN_NO_ERROR;
603 bdb_env_key_t key;
604 bdb_env_t *bdb;
605 svn_boolean_t panic;
607 acquire_cache_mutex();
609 /* We can safely discard the open DB_CONFIG file handle. If the
610 environment descriptor is in the cache, the key's immutability is
611 guaranteed. If it's not, we don't care if the key changes,
612 between here and the actual insertion of the newly-created
613 environment into the cache, because no other thread can touch the
614 cache in the meantime. */
615 err = bdb_cache_key(&key, NULL, path, pool);
616 if (err)
618 release_cache_mutex();
619 return err;
622 bdb = bdb_cache_get(&key, &panic);
623 if (panic)
625 release_cache_mutex();
626 return svn_error_create(SVN_ERR_FS_BERKELEY_DB, NULL,
627 db_strerror(DB_RUNRECOVERY));
630 /* Make sure that the environment's open flags haven't changed. */
631 if (bdb && bdb->flags != flags)
633 release_cache_mutex();
635 /* Handle changes to the DB_PRIVATE flag specially */
636 if ((flags ^ bdb->flags) & DB_PRIVATE)
638 if (flags & DB_PRIVATE)
639 return svn_error_create(SVN_ERR_FS_BERKELEY_DB, NULL,
640 "Reopening a public Berkeley DB"
641 " environment with private attributes");
642 else
643 return svn_error_create(SVN_ERR_FS_BERKELEY_DB, NULL,
644 "Reopening a private Berkeley DB"
645 " environment with public attributes");
648 /* Otherwise return a generic "flags-mismatch" error. */
649 return svn_error_create(SVN_ERR_FS_BERKELEY_DB, NULL,
650 "Reopening a Berkeley DB environment"
651 " with different attributes");
654 if (!bdb)
656 err = create_env(&bdb, path, svn_pool_create(bdb_cache_pool));
657 if (!err)
659 err = bdb_open(bdb, flags, mode);
660 if (!err)
662 apr_hash_set(bdb_cache, &bdb->key, sizeof bdb->key, bdb);
663 bdb->flags = flags;
664 bdb->refcount = 1;
666 else
668 /* Clean up, and we can't do anything about returned errors. */
669 svn_error_clear(bdb_close(bdb));
673 else
675 ++bdb->refcount;
678 if (!err)
680 *bdb_batonp = apr_palloc(pool, sizeof **bdb_batonp);
681 (*bdb_batonp)->env = bdb->env;
682 (*bdb_batonp)->bdb = bdb;
683 (*bdb_batonp)->error_info = get_error_info(bdb);
684 ++(*bdb_batonp)->error_info->refcount;
685 apr_pool_cleanup_register(pool, *bdb_batonp, cleanup_env_baton,
686 apr_pool_cleanup_null);
689 release_cache_mutex();
690 return err;
694 svn_boolean_t
695 svn_fs_bdb__get_panic(bdb_env_baton_t *bdb_baton)
697 /* An invalid baton is equivalent to a panicked environment; in both
698 cases, database cleanups should be skipped. */
699 if (!bdb_baton->bdb)
700 return TRUE;
702 assert(bdb_baton->env == bdb_baton->bdb->env);
703 return !!svn_atomic_read(&bdb_baton->bdb->panic);
706 void
707 svn_fs_bdb__set_panic(bdb_env_baton_t *bdb_baton)
709 if (!bdb_baton->bdb)
710 return;
712 assert(bdb_baton->env == bdb_baton->bdb->env);
713 svn_atomic_set(&bdb_baton->bdb->panic, TRUE);
717 /* This function doesn't actually open the environment, so it doesn't
718 have to look in the cache. Callers are supposed to own an
719 exclusive lock on the filesystem anyway. */
720 svn_error_t *
721 svn_fs_bdb__remove(const char *path, apr_pool_t *pool)
723 bdb_env_t *bdb;
725 SVN_ERR(create_env(&bdb, path, pool));
726 return convert_bdb_error
727 (bdb, bdb->env->remove(bdb->env, bdb->path_bdb, DB_FORCE));