1 /* caching.c : in-memory caching
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
20 * ====================================================================
26 #include "dag_cache.h"
30 #include "temp_serializer.h"
32 #include "../libsvn_fs/fs-loader.h"
34 #include "svn_config.h"
35 #include "svn_cache_config.h"
37 #include "svn_private_config.h"
39 #include "svn_pools.h"
41 #include "private/svn_debug.h"
42 #include "private/svn_subr_private.h"
44 /* Take the ORIGINAL string and replace all occurrences of ":" without
45 * limiting the key space. Allocate the result in RESULT_POOL.
48 normalize_key_part(const char *original
,
49 apr_pool_t
*result_pool
)
52 apr_size_t len
= strlen(original
);
53 svn_stringbuf_t
*normalized
= svn_stringbuf_create_ensure(len
,
56 for (i
= 0; i
< len
; ++i
)
61 case ':': svn_stringbuf_appendbytes(normalized
, "%_", 2);
63 case '%': svn_stringbuf_appendbytes(normalized
, "%%", 2);
65 default : svn_stringbuf_appendbyte(normalized
, c
);
69 return normalized
->data
;
72 /* *CACHE_TXDELTAS, *CACHE_FULLTEXTS, *CACHE_REVPROPS and *CACHE_NODEPROPS
73 flags will be set according to FS->CONFIG. *CACHE_NAMESPACE receives
74 the cache prefix to use.
76 Allocate CACHE_NAMESPACE in RESULT_POOL. */
78 read_config(const char **cache_namespace
,
79 svn_boolean_t
*cache_txdeltas
,
80 svn_boolean_t
*cache_fulltexts
,
81 svn_boolean_t
*cache_revprops
,
82 svn_boolean_t
*cache_nodeprops
,
84 apr_pool_t
*result_pool
)
86 /* No cache namespace by default. I.e. all FS instances share the
87 * cached data. If you specify different namespaces, the data will
88 * share / compete for the same cache memory but keys will not match
89 * across namespaces and, thus, cached data will not be shared between
92 * Since the namespace will be concatenated with other elements to form
93 * the complete key prefix, we must make sure that the resulting string
94 * is unique and cannot be created by any other combination of elements.
97 = normalize_key_part(svn_hash__get_cstring(fs
->config
,
98 SVN_FS_CONFIG_FSFS_CACHE_NS
,
102 /* don't cache text deltas by default.
103 * Once we reconstructed the fulltexts from the deltas,
104 * these deltas are rarely re-used. Therefore, only tools
105 * like svnadmin will activate this to speed up operations
109 = svn_hash__get_bool(fs
->config
,
110 SVN_FS_CONFIG_FSFS_CACHE_DELTAS
,
113 /* by default, cache fulltexts.
114 * Most SVN tools care about reconstructed file content.
115 * Thus, this is a reasonable default.
116 * SVN admin tools may set that to FALSE because fulltexts
117 * won't be re-used rendering the cache less effective
118 * by squeezing wanted data out.
121 = svn_hash__get_bool(fs
->config
,
122 SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS
,
125 /* don't cache revprops by default.
126 * Revprop caching significantly speeds up operations like
127 * svn ls -v. However, it requires synchronization that may
128 * not be available or efficient in the current server setup.
129 * Option "2" is equivalent to "1".
131 if (strcmp(svn_hash__get_cstring(fs
->config
,
132 SVN_FS_CONFIG_FSFS_CACHE_REVPROPS
,
135 = svn_hash__get_bool(fs
->config
,
136 SVN_FS_CONFIG_FSFS_CACHE_REVPROPS
,
139 *cache_revprops
= TRUE
;
141 /* by default, cache nodeprops: this will match pre-1.10
142 * behavior where node properties caching was controlled
143 * by SVN_FS_CONFIG_FSFS_CACHE_FULLTEXTS configuration option.
146 = svn_hash__get_bool(fs
->config
,
147 SVN_FS_CONFIG_FSFS_CACHE_NODEPROPS
,
154 /* Implements svn_cache__error_handler_t
155 * This variant clears the error after logging it.
158 warn_and_continue_on_cache_errors(svn_error_t
*err
,
162 svn_fs_t
*fs
= baton
;
163 (fs
->warning
)(fs
->warning_baton
, err
);
164 svn_error_clear(err
);
169 /* Implements svn_cache__error_handler_t
170 * This variant logs the error and passes it on to the callers.
173 warn_and_fail_on_cache_errors(svn_error_t
*err
,
177 svn_fs_t
*fs
= baton
;
178 (fs
->warning
)(fs
->warning_baton
, err
);
182 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
183 /* Baton to be used for the dump_cache_statistics() pool cleanup function, */
184 typedef struct dump_cache_baton_t
186 /* the pool about to be cleaned up. Will be used for temp. allocations. */
189 /* the cache to dump the statistics for */
191 } dump_cache_baton_t
;
193 /* APR pool cleanup handler that will printf the statistics of the
194 cache referenced by the baton in BATON_VOID. */
196 dump_cache_statistics(void *baton_void
)
198 dump_cache_baton_t
*baton
= baton_void
;
200 apr_status_t result
= APR_SUCCESS
;
201 svn_cache__info_t info
;
202 svn_string_t
*text_stats
;
203 apr_array_header_t
*lines
;
206 svn_error_t
*err
= svn_cache__get_info(baton
->cache
,
211 /* skip unused caches */
212 if (! err
&& (info
.gets
> 0 || info
.sets
> 0))
214 text_stats
= svn_cache__format_info(&info
, TRUE
, baton
->pool
);
215 lines
= svn_cstring_split(text_stats
->data
, "\n", FALSE
, baton
->pool
);
217 for (i
= 0; i
< lines
->nelts
; ++i
)
219 const char *line
= APR_ARRAY_IDX(lines
, i
, const char *);
221 SVN_DBG(("%s\n", line
));
226 /* process error returns */
229 result
= err
->apr_err
;
230 svn_error_clear(err
);
237 dump_global_cache_statistics(void *baton_void
)
239 apr_pool_t
*pool
= baton_void
;
241 svn_cache__info_t
*info
= svn_cache__membuffer_get_global_info(pool
);
242 svn_string_t
*text_stats
= svn_cache__format_info(info
, FALSE
, pool
);
243 apr_array_header_t
*lines
= svn_cstring_split(text_stats
->data
, "\n",
247 for (i
= 0; i
< lines
->nelts
; ++i
)
249 const char *line
= APR_ARRAY_IDX(lines
, i
, const char *);
251 SVN_DBG(("%s\n", line
));
258 #endif /* SVN_DEBUG_CACHE_DUMP_STATS */
260 /* This function sets / registers the required callbacks for a given
261 * not transaction-specific CACHE object in FS, if CACHE is not NULL.
263 * All these svn_cache__t instances shall be handled uniformly. Unless
264 * ERROR_HANDLER is NULL, register it for the given CACHE in FS.
267 init_callbacks(svn_cache__t
*cache
,
269 svn_cache__error_handler_t error_handler
,
272 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
274 /* schedule printing the access statistics upon pool cleanup,
275 * i.e. end of FSX session.
277 dump_cache_baton_t
*baton
;
279 baton
= apr_palloc(pool
, sizeof(*baton
));
281 baton
->cache
= cache
;
283 apr_pool_cleanup_register(pool
,
285 dump_cache_statistics
,
286 apr_pool_cleanup_null
);
290 SVN_ERR(svn_cache__set_error_handler(cache
,
298 /* Sets *CACHE_P to cache instance based on provided options.
300 * If DUMMY_CACHE is set, create a null cache. Otherwise, creates a memcache
301 * if MEMCACHE is not NULL or a membuffer cache if MEMBUFFER is not NULL.
302 * Falls back to inprocess cache if no other cache type has been selected
303 * and PAGES is not 0. Create a null cache otherwise.
305 * Use the given PRIORITY class for the new cache. If PRIORITY is 0, then
306 * use the default priority class. HAS_NAMESPACE indicates whether we
307 * prefixed this cache instance with a namespace.
309 * Unless NO_HANDLER is true, register an error handler that reports errors
310 * as warnings to the FS warning callback.
312 * Cache is allocated in RESULT_POOL, temporaries in SCRATCH_POOL.
315 create_cache(svn_cache__t
**cache_p
,
316 svn_memcache_t
*memcache
,
317 svn_membuffer_t
*membuffer
,
319 apr_int64_t items_per_page
,
320 svn_cache__serialize_func_t serializer
,
321 svn_cache__deserialize_func_t deserializer
,
324 apr_uint32_t priority
,
325 svn_boolean_t has_namespace
,
327 svn_boolean_t no_handler
,
328 svn_boolean_t dummy_cache
,
329 apr_pool_t
*result_pool
,
330 apr_pool_t
*scratch_pool
)
332 svn_cache__error_handler_t error_handler
= no_handler
334 : warn_and_fail_on_cache_errors
;
336 priority
= SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY
;
340 SVN_ERR(svn_cache__create_null(cache_p
, prefix
, result_pool
));
344 SVN_ERR(svn_cache__create_memcache(cache_p
, memcache
,
345 serializer
, deserializer
, klen
,
346 prefix
, result_pool
));
347 error_handler
= no_handler
349 : warn_and_continue_on_cache_errors
;
353 /* We assume caches with namespaces to be relatively short-lived,
354 * i.e. their data will not be needed after a while. */
355 SVN_ERR(svn_cache__create_membuffer_cache(
356 cache_p
, membuffer
, serializer
, deserializer
,
357 klen
, prefix
, priority
, FALSE
, has_namespace
,
358 result_pool
, scratch_pool
));
362 SVN_ERR(svn_cache__create_inprocess(
363 cache_p
, serializer
, deserializer
, klen
, pages
,
364 items_per_page
, FALSE
, prefix
, result_pool
));
368 SVN_ERR(svn_cache__create_null(cache_p
, prefix
, result_pool
));
371 SVN_ERR(init_callbacks(*cache_p
, fs
, error_handler
, result_pool
));
377 svn_fs_x__initialize_caches(svn_fs_t
*fs
,
378 apr_pool_t
*scratch_pool
)
380 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
381 const char *prefix
= apr_pstrcat(scratch_pool
,
383 "--", ffd
->instance_id
,
384 "/", normalize_key_part(fs
->path
,
388 svn_membuffer_t
*membuffer
;
389 svn_boolean_t no_handler
= ffd
->fail_stop
;
390 svn_boolean_t cache_txdeltas
;
391 svn_boolean_t cache_fulltexts
;
392 svn_boolean_t cache_revprops
;
393 svn_boolean_t cache_nodeprops
;
394 const char *cache_namespace
;
395 svn_boolean_t has_namespace
;
397 /* Evaluating the cache configuration. */
398 SVN_ERR(read_config(&cache_namespace
,
406 prefix
= apr_pstrcat(scratch_pool
, "ns:", cache_namespace
, ":", prefix
,
408 has_namespace
= strlen(cache_namespace
) > 0;
410 membuffer
= svn_cache__get_global_membuffer_cache();
412 /* General rules for assigning cache priorities:
414 * - Data that can be reconstructed from other elements has low prio
415 * (e.g. fulltexts etc.)
416 * - Index data required to find any of the other data has high prio
417 * (e.g. noderevs, L2P and P2L index pages)
418 * - everthing else should use default prio
421 #ifdef SVN_DEBUG_CACHE_DUMP_STATS
423 /* schedule printing the global access statistics upon pool cleanup,
424 * i.e. end of FSX session.
427 apr_pool_cleanup_register(fs
->pool
,
429 dump_global_cache_statistics
,
430 apr_pool_cleanup_null
);
433 /* 1st level DAG node cache */
434 ffd
->dag_node_cache
= svn_fs_x__create_dag_cache(fs
->pool
);
436 /* Very rough estimate: 1K per directory. */
437 SVN_ERR(create_cache(&(ffd
->dir_cache
),
441 svn_fs_x__serialize_dir_entries
,
442 svn_fs_x__deserialize_dir_entries
,
443 sizeof(svn_fs_x__id_t
),
444 apr_pstrcat(scratch_pool
, prefix
, "DIR", SVN_VA_NULL
),
445 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY
,
449 fs
->pool
, scratch_pool
));
451 /* initialize node revision cache, if caching has been enabled */
452 SVN_ERR(create_cache(&(ffd
->node_revision_cache
),
455 32, 32, /* ~200 byte / entry; 1k entries total */
456 svn_fs_x__serialize_node_revision
,
457 svn_fs_x__deserialize_node_revision
,
458 sizeof(svn_fs_x__pair_cache_key_t
),
459 apr_pstrcat(scratch_pool
, prefix
, "NODEREVS",
461 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY
,
465 fs
->pool
, scratch_pool
));
467 /* initialize representation header cache, if caching has been enabled */
468 SVN_ERR(create_cache(&(ffd
->rep_header_cache
),
471 1, 1000, /* ~8 bytes / entry; 1k entries total */
472 svn_fs_x__serialize_rep_header
,
473 svn_fs_x__deserialize_rep_header
,
474 sizeof(svn_fs_x__representation_cache_key_t
),
475 apr_pstrcat(scratch_pool
, prefix
, "REPHEADER",
477 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY
,
481 fs
->pool
, scratch_pool
));
483 /* initialize node change list cache, if caching has been enabled */
484 SVN_ERR(create_cache(&(ffd
->changes_cache
),
487 1, 8, /* 1k / entry; 8 entries total, rarely used */
488 svn_fs_x__serialize_changes
,
489 svn_fs_x__deserialize_changes
,
490 sizeof(svn_fs_x__pair_cache_key_t
),
491 apr_pstrcat(scratch_pool
, prefix
, "CHANGES",
497 fs
->pool
, scratch_pool
));
499 /* if enabled, cache fulltext and other derived information */
500 SVN_ERR(create_cache(&(ffd
->fulltext_cache
),
503 0, 0, /* Do not use inprocess cache */
504 /* Values are svn_stringbuf_t */
506 sizeof(svn_fs_x__pair_cache_key_t
),
507 apr_pstrcat(scratch_pool
, prefix
, "TEXT",
509 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY
,
512 no_handler
, !cache_fulltexts
,
513 fs
->pool
, scratch_pool
));
515 SVN_ERR(create_cache(&(ffd
->properties_cache
),
518 0, 0, /* Do not use inprocess cache */
519 svn_fs_x__serialize_properties
,
520 svn_fs_x__deserialize_properties
,
521 sizeof(svn_fs_x__pair_cache_key_t
),
522 apr_pstrcat(scratch_pool
, prefix
, "PROP",
524 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY
,
527 no_handler
, !cache_nodeprops
,
528 fs
->pool
, scratch_pool
));
530 /* if enabled, cache revprops */
531 SVN_ERR(create_cache(&(ffd
->revprop_cache
),
534 0, 0, /* Do not use inprocess cache */
535 svn_fs_x__serialize_properties
,
536 svn_fs_x__deserialize_properties
,
537 sizeof(svn_fs_x__pair_cache_key_t
),
538 apr_pstrcat(scratch_pool
, prefix
, "REVPROP",
540 SVN_CACHE__MEMBUFFER_DEFAULT_PRIORITY
,
543 no_handler
, !cache_revprops
,
544 fs
->pool
, scratch_pool
));
546 /* if enabled, cache text deltas and their combinations */
547 SVN_ERR(create_cache(&(ffd
->txdelta_window_cache
),
550 0, 0, /* Do not use inprocess cache */
551 svn_fs_x__serialize_txdelta_window
,
552 svn_fs_x__deserialize_txdelta_window
,
553 sizeof(svn_fs_x__window_cache_key_t
),
554 apr_pstrcat(scratch_pool
, prefix
, "TXDELTA_WINDOW",
556 SVN_CACHE__MEMBUFFER_LOW_PRIORITY
,
559 no_handler
, !cache_txdeltas
,
560 fs
->pool
, scratch_pool
));
562 SVN_ERR(create_cache(&(ffd
->combined_window_cache
),
565 0, 0, /* Do not use inprocess cache */
566 /* Values are svn_stringbuf_t */
568 sizeof(svn_fs_x__window_cache_key_t
),
569 apr_pstrcat(scratch_pool
, prefix
, "COMBINED_WINDOW",
571 SVN_CACHE__MEMBUFFER_LOW_PRIORITY
,
574 no_handler
, !cache_txdeltas
,
575 fs
->pool
, scratch_pool
));
577 /* Caches for our various container types. */
578 SVN_ERR(create_cache(&(ffd
->noderevs_container_cache
),
581 16, 4, /* Important, largish objects */
582 svn_fs_x__serialize_noderevs_container
,
583 svn_fs_x__deserialize_noderevs_container
,
584 sizeof(svn_fs_x__pair_cache_key_t
),
585 apr_pstrcat(scratch_pool
, prefix
, "NODEREVSCNT",
587 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY
,
591 fs
->pool
, scratch_pool
));
592 SVN_ERR(create_cache(&(ffd
->changes_container_cache
),
595 0, 0, /* Do not use inprocess cache */
596 svn_fs_x__serialize_changes_container
,
597 svn_fs_x__deserialize_changes_container
,
598 sizeof(svn_fs_x__pair_cache_key_t
),
599 apr_pstrcat(scratch_pool
, prefix
, "CHANGESCNT",
605 fs
->pool
, scratch_pool
));
606 SVN_ERR(create_cache(&(ffd
->reps_container_cache
),
609 0, 0, /* Do not use inprocess cache */
610 svn_fs_x__serialize_reps_container
,
611 svn_fs_x__deserialize_reps_container
,
612 sizeof(svn_fs_x__pair_cache_key_t
),
613 apr_pstrcat(scratch_pool
, prefix
, "REPSCNT",
619 fs
->pool
, scratch_pool
));
621 /* Cache index info. */
622 SVN_ERR(create_cache(&(ffd
->l2p_header_cache
),
625 64, 16, /* entry size varies but we must cover
626 a reasonable number of revisions (1k) */
627 svn_fs_x__serialize_l2p_header
,
628 svn_fs_x__deserialize_l2p_header
,
629 sizeof(svn_fs_x__pair_cache_key_t
),
630 apr_pstrcat(scratch_pool
, prefix
, "L2P_HEADER",
632 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY
,
636 fs
->pool
, scratch_pool
));
637 SVN_ERR(create_cache(&(ffd
->l2p_page_cache
),
640 64, 16, /* entry size varies but we must cover
641 a reasonable number of revisions (1k) */
642 svn_fs_x__serialize_l2p_page
,
643 svn_fs_x__deserialize_l2p_page
,
644 sizeof(svn_fs_x__page_cache_key_t
),
645 apr_pstrcat(scratch_pool
, prefix
, "L2P_PAGE",
647 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY
,
651 fs
->pool
, scratch_pool
));
652 SVN_ERR(create_cache(&(ffd
->p2l_header_cache
),
655 4, 1, /* Large entries. Rarely used. */
656 svn_fs_x__serialize_p2l_header
,
657 svn_fs_x__deserialize_p2l_header
,
658 sizeof(svn_fs_x__pair_cache_key_t
),
659 apr_pstrcat(scratch_pool
, prefix
, "P2L_HEADER",
661 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY
,
665 fs
->pool
, scratch_pool
));
666 SVN_ERR(create_cache(&(ffd
->p2l_page_cache
),
669 4, 16, /* Variably sized entries. Rarely used. */
670 svn_fs_x__serialize_p2l_page
,
671 svn_fs_x__deserialize_p2l_page
,
672 sizeof(svn_fs_x__page_cache_key_t
),
673 apr_pstrcat(scratch_pool
, prefix
, "P2L_PAGE",
675 SVN_CACHE__MEMBUFFER_HIGH_PRIORITY
,
679 fs
->pool
, scratch_pool
));