Win32: fix an incorrect error status being propagated to the caller in case
[svn/apache.git] / subversion / libsvn_fs_x / caching.c
blob8c65ab11a2a84c9450765e40dd481b67c3e19782
1 /* caching.c : in-memory caching
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
19 * under the License.
20 * ====================================================================
23 #include "fs.h"
24 #include "fs_x.h"
25 #include "id.h"
26 #include "dag_cache.h"
27 #include "index.h"
28 #include "changes.h"
29 #include "noderevs.h"
30 #include "temp_serializer.h"
31 #include "reps.h"
32 #include "../libsvn_fs/fs-loader.h"
34 #include "svn_config.h"
35 #include "svn_cache_config.h"
37 #include "svn_private_config.h"
38 #include "svn_hash.h"
39 #include "svn_pools.h"
41 #include "private/svn_debug.h"
42 #include "private/svn_subr_private.h"
44 /* Take the ORIGINAL string and replace all occurrences of ":" without
45 * limiting the key space. Allocate the result in RESULT_POOL.
47 static const char *
48 normalize_key_part(const char *original,
49 apr_pool_t *result_pool)
51 apr_size_t i;
52 apr_size_t len = strlen(original);
53 svn_stringbuf_t *normalized = svn_stringbuf_create_ensure(len,
54 result_pool);
56 for (i = 0; i < len; ++i)
58 char c = original[i];
59 switch (c)
61 case ':': svn_stringbuf_appendbytes(normalized, "%_", 2);
62 break;
63 case '%': svn_stringbuf_appendbytes(normalized, "%%", 2);
64 break;
65 default : svn_stringbuf_appendbyte(normalized, c);
69 return normalized->data;
72 /* *CACHE_TXDELTAS, *CACHE_FULLTEXTS, *CACHE_REVPROPS and *CACHE_NODEPROPS
73 flags will be set according to FS->CONFIG. *CACHE_NAMESPACE receives
74 the cache prefix to use.
76 Allocate CACHE_NAMESPACE in RESULT_POOL. */
77 static svn_error_t *
78 read_config(const char **cache_namespace,
79 svn_boolean_t *cache_txdeltas,
80 svn_boolean_t *cache_fulltexts,
81 svn_boolean_t *cache_revprops,
82 svn_boolean_t *cache_nodeprops,
83 svn_fs_t *fs,
84 apr_pool_t *result_pool)
86 /* No cache namespace by default. I.e. all FS instances share the
87 * cached data. If you specify different namespaces, the data will
88 * share / compete for the same cache memory but keys will not match
89 * across namespaces and, thus, cached data will not be shared between
90 * namespaces.
92 * Since the namespace will be concatenated with other elements to form
93 * the complete key prefix, we must make sure that the resulting string
94 * is unique and cannot be created by any other combination of elements.
96 *cache_namespace
97 = normalize_key_part(svn_hash__get_cstring(fs->config,
98 SVN_FS_CONFIG_FSFS_CACHE_NS,
99 ""),
100 result_pool);
102 /* don't cache text deltas by default.
103 * Once we reconstructed the fulltexts from the deltas,
104 * these deltas are rarely re-used. Therefore, only tools
105 * like svnadmin will activate this to speed up operations
106 * dump and verify.
108 *cache_txdeltas
109 = svn_hash__get_bool(fs->config,
110 SVN_FS_CONFIG_FSFS_CACHE_DELTAS,
111 TRUE);
113 /* by default, cache fulltexts.
114 * Most SVN tools care about reconstructed file content.
115 * Thus, this is a reasonable default.
116 * SVN admin tools may set that to FALSE because fulltexts
117 * won't be re-used rendering the cache less effective
118 * by squeezing wanted data out.
120 *cache_fulltexts
121 = svn_hash__get_bool(fs->config,
122 SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS,
123 TRUE);
125 /* don't cache revprops by default.
126 * Revprop caching significantly speeds up operations like
127 * svn ls -v. However, it requires synchronization that may
128 * not be available or efficient in the current server setup.
129 * Option "2" is equivalent to "1".
131 if (strcmp(svn_hash__get_cstring(fs->config,
132 SVN_FS_CONFIG_FSFS_CACHE_REVPROPS,
133 ""), "2"))
134 *cache_revprops
135 = svn_hash__get_bool(fs->config,
136 SVN_FS_CONFIG_FSFS_CACHE_REVPROPS,
137 FALSE);
138 else
139 *cache_revprops = TRUE;
141 /* by default, cache nodeprops: this will match pre-1.10
142 * behavior where node properties caching was controlled
143 * by SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS configuration option.
145 *cache_nodeprops
146 = svn_hash__get_bool(fs->config,
147 SVN_FS_CONFIG_FSFS_CACHE_NODEPROPS,
148 TRUE);
150 return SVN_NO_ERROR;
154 /* Implements svn_cache__error_handler_t
155 * This variant clears the error after logging it.
157 static svn_error_t *
158 warn_and_continue_on_cache_errors(svn_error_t *err,
159 void *baton,
160 apr_pool_t *pool)
162 svn_fs_t *fs = baton;
163 (fs->warning)(fs->warning_baton, err);
164 svn_error_clear(err);
166 return SVN_NO_ERROR;
169 /* Implements svn_cache__error_handler_t
170 * This variant logs the error and passes it on to the callers.
172 static svn_error_t *
173 warn_and_fail_on_cache_errors(svn_error_t *err,
174 void *baton,
175 apr_pool_t *pool)
177 svn_fs_t *fs = baton;
178 (fs->warning)(fs->warning_baton, err);
179 return err;
182 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
183 /* Baton to be used for the dump_cache_statistics() pool cleanup function, */
184 typedef struct dump_cache_baton_t
186 /* the pool about to be cleaned up. Will be used for temp. allocations. */
187 apr_pool_t *pool;
189 /* the cache to dump the statistics for */
190 svn_cache__t *cache;
191 } dump_cache_baton_t;
193 /* APR pool cleanup handler that will printf the statistics of the
194 cache referenced by the baton in BATON_VOID. */
195 static apr_status_t
196 dump_cache_statistics(void *baton_void)
198 dump_cache_baton_t *baton = baton_void;
200 apr_status_t result = APR_SUCCESS;
201 svn_cache__info_t info;
202 svn_string_t *text_stats;
203 apr_array_header_t *lines;
204 int i;
206 svn_error_t *err = svn_cache__get_info(baton->cache,
207 &info,
208 TRUE,
209 baton->pool);
211 /* skip unused caches */
212 if (! err && (info.gets > 0 || info.sets > 0))
214 text_stats = svn_cache__format_info(&info, TRUE, baton->pool);
215 lines = svn_cstring_split(text_stats->data, "\n", FALSE, baton->pool);
217 for (i = 0; i < lines->nelts; ++i)
219 const char *line = APR_ARRAY_IDX(lines, i, const char *);
220 #ifdef SVN_DEBUG
221 SVN_DBG(("%s\n", line));
222 #endif
226 /* process error returns */
227 if (err)
229 result = err->apr_err;
230 svn_error_clear(err);
233 return result;
236 static apr_status_t
237 dump_global_cache_statistics(void *baton_void)
239 apr_pool_t *pool = baton_void;
241 svn_cache__info_t *info = svn_cache__membuffer_get_global_info(pool);
242 svn_string_t *text_stats = svn_cache__format_info(info, FALSE, pool);
243 apr_array_header_t *lines = svn_cstring_split(text_stats->data, "\n",
244 FALSE, pool);
246 int i;
247 for (i = 0; i < lines->nelts; ++i)
249 const char *line = APR_ARRAY_IDX(lines, i, const char *);
250 #ifdef SVN_DEBUG
251 SVN_DBG(("%s\n", line));
252 #endif
255 return APR_SUCCESS;
258 #endif /* SVN_DEBUG_CACHE_DUMP_STATS */
260 /* This function sets / registers the required callbacks for a given
261 * not transaction-specific CACHE object in FS, if CACHE is not NULL.
263 * All these svn_cache__t instances shall be handled uniformly. Unless
264 * ERROR_HANDLER is NULL, register it for the given CACHE in FS.
266 static svn_error_t *
267 init_callbacks(svn_cache__t *cache,
268 svn_fs_t *fs,
269 svn_cache__error_handler_t error_handler,
270 apr_pool_t *pool)
272 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
274 /* schedule printing the access statistics upon pool cleanup,
275 * i.e. end of FSX session.
277 dump_cache_baton_t *baton;
279 baton = apr_palloc(pool, sizeof(*baton));
280 baton->pool = pool;
281 baton->cache = cache;
283 apr_pool_cleanup_register(pool,
284 baton,
285 dump_cache_statistics,
286 apr_pool_cleanup_null);
287 #endif
289 if (error_handler)
290 SVN_ERR(svn_cache__set_error_handler(cache,
291 error_handler,
293 pool));
295 return SVN_NO_ERROR;
298 /* Sets *CACHE_P to cache instance based on provided options.
300 * If DUMMY_CACHE is set, create a null cache. Otherwise, creates a memcache
301 * if MEMCACHE is not NULL or a membuffer cache if MEMBUFFER is not NULL.
302 * Falls back to inprocess cache if no other cache type has been selected
303 * and PAGES is not 0. Create a null cache otherwise.
305 * Use the given PRIORITY class for the new cache. If PRIORITY is 0, then
306 * use the default priority class. HAS_NAMESPACE indicates whether we
307 * prefixed this cache instance with a namespace.
309 * Unless NO_HANDLER is true, register an error handler that reports errors
310 * as warnings to the FS warning callback.
312 * Cache is allocated in RESULT_POOL, temporaries in SCRATCH_POOL.
313 * */
314 static svn_error_t *
315 create_cache(svn_cache__t **cache_p,
316 svn_memcache_t *memcache,
317 svn_membuffer_t *membuffer,
318 apr_int64_t pages,
319 apr_int64_t items_per_page,
320 svn_cache__serialize_func_t serializer,
321 svn_cache__deserialize_func_t deserializer,
322 apr_ssize_t klen,
323 const char *prefix,
324 apr_uint32_t priority,
325 svn_boolean_t has_namespace,
326 svn_fs_t *fs,
327 svn_boolean_t no_handler,
328 svn_boolean_t dummy_cache,
329 apr_pool_t *result_pool,
330 apr_pool_t *scratch_pool)
332 svn_cache__error_handler_t error_handler = no_handler
333 ? NULL
334 : warn_and_fail_on_cache_errors;
335 if (priority == 0)
336 priority = SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY;
338 if (dummy_cache)
340 SVN_ERR(svn_cache__create_null(cache_p, prefix, result_pool));
342 else if (memcache)
344 SVN_ERR(svn_cache__create_memcache(cache_p, memcache,
345 serializer, deserializer, klen,
346 prefix, result_pool));
347 error_handler = no_handler
348 ? NULL
349 : warn_and_continue_on_cache_errors;
351 else if (membuffer)
353 /* We assume caches with namespaces to be relatively short-lived,
354 * i.e. their data will not be needed after a while. */
355 SVN_ERR(svn_cache__create_membuffer_cache(
356 cache_p, membuffer, serializer, deserializer,
357 klen, prefix, priority, FALSE, has_namespace,
358 result_pool, scratch_pool));
360 else if (pages)
362 SVN_ERR(svn_cache__create_inprocess(
363 cache_p, serializer, deserializer, klen, pages,
364 items_per_page, FALSE, prefix, result_pool));
366 else
368 SVN_ERR(svn_cache__create_null(cache_p, prefix, result_pool));
371 SVN_ERR(init_callbacks(*cache_p, fs, error_handler, result_pool));
373 return SVN_NO_ERROR;
376 svn_error_t *
377 svn_fs_x__initialize_caches(svn_fs_t *fs,
378 apr_pool_t *scratch_pool)
380 svn_fs_x__data_t *ffd = fs->fsap_data;
381 const char *prefix = apr_pstrcat(scratch_pool,
382 "fsx:", fs->uuid,
383 "--", ffd->instance_id,
384 "/", normalize_key_part(fs->path,
385 scratch_pool),
386 ":",
387 SVN_VA_NULL);
388 svn_membuffer_t *membuffer;
389 svn_boolean_t no_handler = ffd->fail_stop;
390 svn_boolean_t cache_txdeltas;
391 svn_boolean_t cache_fulltexts;
392 svn_boolean_t cache_revprops;
393 svn_boolean_t cache_nodeprops;
394 const char *cache_namespace;
395 svn_boolean_t has_namespace;
397 /* Evaluating the cache configuration. */
398 SVN_ERR(read_config(&cache_namespace,
399 &cache_txdeltas,
400 &cache_fulltexts,
401 &cache_revprops,
402 &cache_nodeprops,
404 scratch_pool));
406 prefix = apr_pstrcat(scratch_pool, "ns:", cache_namespace, ":", prefix,
407 SVN_VA_NULL);
408 has_namespace = strlen(cache_namespace) > 0;
410 membuffer = svn_cache__get_global_membuffer_cache();
412 /* General rules for assigning cache priorities:
414 * - Data that can be reconstructed from other elements has low prio
415 * (e.g. fulltexts etc.)
416 * - Index data required to find any of the other data has high prio
417 * (e.g. noderevs, L2P and P2L index pages)
418 * - everthing else should use default prio
421 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
423 /* schedule printing the global access statistics upon pool cleanup,
424 * i.e. end of FSX session.
426 if (membuffer)
427 apr_pool_cleanup_register(fs->pool,
428 fs->pool,
429 dump_global_cache_statistics,
430 apr_pool_cleanup_null);
431 #endif
433 /* 1st level DAG node cache */
434 ffd->dag_node_cache = svn_fs_x__create_dag_cache(fs->pool);
436 /* Very rough estimate: 1K per directory. */
437 SVN_ERR(create_cache(&(ffd->dir_cache),
438 NULL,
439 membuffer,
440 1024, 8,
441 svn_fs_x__serialize_dir_entries,
442 svn_fs_x__deserialize_dir_entries,
443 sizeof(svn_fs_x__id_t),
444 apr_pstrcat(scratch_pool, prefix, "DIR", SVN_VA_NULL),
445 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
446 has_namespace,
448 no_handler, FALSE,
449 fs->pool, scratch_pool));
451 /* initialize node revision cache, if caching has been enabled */
452 SVN_ERR(create_cache(&(ffd->node_revision_cache),
453 NULL,
454 membuffer,
455 32, 32, /* ~200 byte / entry; 1k entries total */
456 svn_fs_x__serialize_node_revision,
457 svn_fs_x__deserialize_node_revision,
458 sizeof(svn_fs_x__pair_cache_key_t),
459 apr_pstrcat(scratch_pool, prefix, "NODEREVS",
460 SVN_VA_NULL),
461 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
462 has_namespace,
464 no_handler, FALSE,
465 fs->pool, scratch_pool));
467 /* initialize representation header cache, if caching has been enabled */
468 SVN_ERR(create_cache(&(ffd->rep_header_cache),
469 NULL,
470 membuffer,
471 1, 1000, /* ~8 bytes / entry; 1k entries total */
472 svn_fs_x__serialize_rep_header,
473 svn_fs_x__deserialize_rep_header,
474 sizeof(svn_fs_x__representation_cache_key_t),
475 apr_pstrcat(scratch_pool, prefix, "REPHEADER",
476 SVN_VA_NULL),
477 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
478 has_namespace,
480 no_handler, FALSE,
481 fs->pool, scratch_pool));
483 /* initialize node change list cache, if caching has been enabled */
484 SVN_ERR(create_cache(&(ffd->changes_cache),
485 NULL,
486 membuffer,
487 1, 8, /* 1k / entry; 8 entries total, rarely used */
488 svn_fs_x__serialize_changes,
489 svn_fs_x__deserialize_changes,
490 sizeof(svn_fs_x__pair_cache_key_t),
491 apr_pstrcat(scratch_pool, prefix, "CHANGES",
492 SVN_VA_NULL),
494 has_namespace,
496 no_handler, FALSE,
497 fs->pool, scratch_pool));
499 /* if enabled, cache fulltext and other derived information */
500 SVN_ERR(create_cache(&(ffd->fulltext_cache),
501 ffd->memcache,
502 membuffer,
503 0, 0, /* Do not use inprocess cache */
504 /* Values are svn_stringbuf_t */
505 NULL, NULL,
506 sizeof(svn_fs_x__pair_cache_key_t),
507 apr_pstrcat(scratch_pool, prefix, "TEXT",
508 SVN_VA_NULL),
509 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
510 has_namespace,
512 no_handler, !cache_fulltexts,
513 fs->pool, scratch_pool));
515 SVN_ERR(create_cache(&(ffd->properties_cache),
516 NULL,
517 membuffer,
518 0, 0, /* Do not use inprocess cache */
519 svn_fs_x__serialize_properties,
520 svn_fs_x__deserialize_properties,
521 sizeof(svn_fs_x__pair_cache_key_t),
522 apr_pstrcat(scratch_pool, prefix, "PROP",
523 SVN_VA_NULL),
524 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
525 has_namespace,
527 no_handler, !cache_nodeprops,
528 fs->pool, scratch_pool));
530 /* if enabled, cache revprops */
531 SVN_ERR(create_cache(&(ffd->revprop_cache),
532 NULL,
533 membuffer,
534 0, 0, /* Do not use inprocess cache */
535 svn_fs_x__serialize_properties,
536 svn_fs_x__deserialize_properties,
537 sizeof(svn_fs_x__pair_cache_key_t),
538 apr_pstrcat(scratch_pool, prefix, "REVPROP",
539 SVN_VA_NULL),
540 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY,
541 has_namespace,
543 no_handler, !cache_revprops,
544 fs->pool, scratch_pool));
546 /* if enabled, cache text deltas and their combinations */
547 SVN_ERR(create_cache(&(ffd->txdelta_window_cache),
548 NULL,
549 membuffer,
550 0, 0, /* Do not use inprocess cache */
551 svn_fs_x__serialize_txdelta_window,
552 svn_fs_x__deserialize_txdelta_window,
553 sizeof(svn_fs_x__window_cache_key_t),
554 apr_pstrcat(scratch_pool, prefix, "TXDELTA_WINDOW",
555 SVN_VA_NULL),
556 SVN_CACHE__MEMBUFFER_LOW_PRIORITY,
557 has_namespace,
559 no_handler, !cache_txdeltas,
560 fs->pool, scratch_pool));
562 SVN_ERR(create_cache(&(ffd->combined_window_cache),
563 NULL,
564 membuffer,
565 0, 0, /* Do not use inprocess cache */
566 /* Values are svn_stringbuf_t */
567 NULL, NULL,
568 sizeof(svn_fs_x__window_cache_key_t),
569 apr_pstrcat(scratch_pool, prefix, "COMBINED_WINDOW",
570 SVN_VA_NULL),
571 SVN_CACHE__MEMBUFFER_LOW_PRIORITY,
572 has_namespace,
574 no_handler, !cache_txdeltas,
575 fs->pool, scratch_pool));
577 /* Caches for our various container types. */
578 SVN_ERR(create_cache(&(ffd->noderevs_container_cache),
579 NULL,
580 membuffer,
581 16, 4, /* Important, largish objects */
582 svn_fs_x__serialize_noderevs_container,
583 svn_fs_x__deserialize_noderevs_container,
584 sizeof(svn_fs_x__pair_cache_key_t),
585 apr_pstrcat(scratch_pool, prefix, "NODEREVSCNT",
586 SVN_VA_NULL),
587 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
588 has_namespace,
590 no_handler, FALSE,
591 fs->pool, scratch_pool));
592 SVN_ERR(create_cache(&(ffd->changes_container_cache),
593 NULL,
594 membuffer,
595 0, 0, /* Do not use inprocess cache */
596 svn_fs_x__serialize_changes_container,
597 svn_fs_x__deserialize_changes_container,
598 sizeof(svn_fs_x__pair_cache_key_t),
599 apr_pstrcat(scratch_pool, prefix, "CHANGESCNT",
600 SVN_VA_NULL),
602 has_namespace,
604 no_handler, FALSE,
605 fs->pool, scratch_pool));
606 SVN_ERR(create_cache(&(ffd->reps_container_cache),
607 NULL,
608 membuffer,
609 0, 0, /* Do not use inprocess cache */
610 svn_fs_x__serialize_reps_container,
611 svn_fs_x__deserialize_reps_container,
612 sizeof(svn_fs_x__pair_cache_key_t),
613 apr_pstrcat(scratch_pool, prefix, "REPSCNT",
614 SVN_VA_NULL),
616 has_namespace,
618 no_handler, FALSE,
619 fs->pool, scratch_pool));
621 /* Cache index info. */
622 SVN_ERR(create_cache(&(ffd->l2p_header_cache),
623 NULL,
624 membuffer,
625 64, 16, /* entry size varies but we must cover
626 a reasonable number of revisions (1k) */
627 svn_fs_x__serialize_l2p_header,
628 svn_fs_x__deserialize_l2p_header,
629 sizeof(svn_fs_x__pair_cache_key_t),
630 apr_pstrcat(scratch_pool, prefix, "L2P_HEADER",
631 SVN_VA_NULL),
632 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
633 has_namespace,
635 no_handler, FALSE,
636 fs->pool, scratch_pool));
637 SVN_ERR(create_cache(&(ffd->l2p_page_cache),
638 NULL,
639 membuffer,
640 64, 16, /* entry size varies but we must cover
641 a reasonable number of revisions (1k) */
642 svn_fs_x__serialize_l2p_page,
643 svn_fs_x__deserialize_l2p_page,
644 sizeof(svn_fs_x__page_cache_key_t),
645 apr_pstrcat(scratch_pool, prefix, "L2P_PAGE",
646 SVN_VA_NULL),
647 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
648 has_namespace,
650 no_handler, FALSE,
651 fs->pool, scratch_pool));
652 SVN_ERR(create_cache(&(ffd->p2l_header_cache),
653 NULL,
654 membuffer,
655 4, 1, /* Large entries. Rarely used. */
656 svn_fs_x__serialize_p2l_header,
657 svn_fs_x__deserialize_p2l_header,
658 sizeof(svn_fs_x__pair_cache_key_t),
659 apr_pstrcat(scratch_pool, prefix, "P2L_HEADER",
660 SVN_VA_NULL),
661 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
662 has_namespace,
664 no_handler, FALSE,
665 fs->pool, scratch_pool));
666 SVN_ERR(create_cache(&(ffd->p2l_page_cache),
667 NULL,
668 membuffer,
669 4, 16, /* Variably sized entries. Rarely used. */
670 svn_fs_x__serialize_p2l_page,
671 svn_fs_x__deserialize_p2l_page,
672 sizeof(svn_fs_x__page_cache_key_t),
673 apr_pstrcat(scratch_pool, prefix, "P2L_PAGE",
674 SVN_VA_NULL),
675 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY,
676 has_namespace,
678 no_handler, FALSE,
679 fs->pool, scratch_pool));
681 return SVN_NO_ERROR;