1 /* fs_x.c --- filesystem operations specific to fs_x
3 * ====================================================================
4 * Licensed to the Apache Software Foundation (ASF) under one
5 * or more contributor license agreements. See the NOTICE file
6 * distributed with this work for additional information
7 * regarding copyright ownership. The ASF licenses this file
8 * to you under the Apache License, Version 2.0 (the
9 * "License"); you may not use this file except in compliance
10 * with the License. You may obtain a copy of the License at
12 * http://www.apache.org/licenses/LICENSE-2.0
14 * Unless required by applicable law or agreed to in writing,
15 * software distributed under the License is distributed on an
16 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
17 * KIND, either express or implied. See the License for the
18 * specific language governing permissions and limitations
20 * ====================================================================
28 #include "svn_props.h"
30 #include "svn_dirent_uri.h"
31 #include "svn_sorts.h"
32 #include "svn_version.h"
34 #include "cached_data.h"
36 #include "low_level.h"
37 #include "rep-cache.h"
39 #include "transaction.h"
44 #include "private/svn_fs_util.h"
45 #include "private/svn_string_private.h"
46 #include "private/svn_subr_private.h"
47 #include "../libsvn_fs/fs-loader.h"
49 #include "svn_private_config.h"
51 /* The default maximum number of files per directory to store in the
52 rev and revprops directory. The number below is somewhat arbitrary,
53 and can be overridden by defining the macro while compiling; the
54 figure of 1000 is reasonable for VFAT filesystems, which are by far
55 the worst performers in this area. */
56 #ifndef SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR
57 #define SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR 1000
60 /* Begin deltification after a node history exceeded this this limit.
61 Useful values are 4 to 64 with 16 being a good compromise between
62 computational overhead and repository size savings.
63 Should be a power of 2.
64 Values < 2 will result in standard skip-delta behavior. */
65 #define SVN_FS_X_MAX_LINEAR_DELTIFICATION 16
67 /* Finding a deltification base takes operations proportional to the
68 number of changes being skipped. To prevent exploding runtime
69 during commits, limit the deltification range to this value.
70 Should be a power of 2 minus one.
71 Values < 1 disable deltification. */
72 #define SVN_FS_X_MAX_DELTIFICATION_WALK 1023
77 /* Check that BUF, a nul-terminated buffer of text from format file PATH,
78 contains only digits at OFFSET and beyond, raising an error if not.
80 Uses SCRATCH_POOL for temporary allocation. */
82 check_format_file_buffer_numeric(const char *buf
,
85 apr_pool_t
*scratch_pool
)
87 return svn_fs_x__check_file_buffer_numeric(buf
, offset
, path
, "Format",
91 /* Return the error SVN_ERR_FS_UNSUPPORTED_FORMAT if FS's format
92 number is not the same as a format number supported by this
95 check_format(int format
)
97 /* Put blacklisted versions here. */
99 /* We support any format if it matches the current format. */
100 if (format
== SVN_FS_X__FORMAT_NUMBER
)
103 /* Experimental formats are only supported if they match the current, but
104 * that case has already been handled. So, reject any experimental format.
106 if (SVN_FS_X__EXPERIMENTAL_FORMAT_NUMBER
>= format
)
107 return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT
, NULL
,
108 _("Unsupported experimental FSX format '%d' found; current format is '%d'"),
109 format
, SVN_FS_X__FORMAT_NUMBER
);
111 /* By default, we will support any non-experimental format released so far.
113 if (format
<= SVN_FS_X__FORMAT_NUMBER
)
116 return svn_error_createf(SVN_ERR_FS_UNSUPPORTED_FORMAT
, NULL
,
117 _("Expected FSX format between '%d' and '%d'; found format '%d'"),
118 SVN_FS_X__EXPERIMENTAL_FORMAT_NUMBER
+ 1, SVN_FS_X__FORMAT_NUMBER
,
122 /* Read the format file at PATH and set *PFORMAT to the format version found
123 * and *MAX_FILES_PER_DIR to the shard size. Use SCRATCH_POOL for temporary
126 read_format(int *pformat
,
127 int *max_files_per_dir
,
129 apr_pool_t
*scratch_pool
)
131 svn_stream_t
*stream
;
132 svn_stringbuf_t
*content
;
133 svn_stringbuf_t
*buf
;
134 svn_boolean_t eos
= FALSE
;
136 SVN_ERR(svn_stringbuf_from_file2(&content
, path
, scratch_pool
));
137 stream
= svn_stream_from_stringbuf(content
, scratch_pool
);
138 SVN_ERR(svn_stream_readline(stream
, &buf
, "\n", &eos
, scratch_pool
));
139 if (buf
->len
== 0 && eos
)
141 /* Return a more useful error message. */
142 return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT
, NULL
,
143 _("Can't read first line of format file '%s'"),
144 svn_dirent_local_style(path
, scratch_pool
));
147 /* Check that the first line contains only digits. */
148 SVN_ERR(check_format_file_buffer_numeric(buf
->data
, 0, path
, scratch_pool
));
149 SVN_ERR(svn_cstring_atoi(pformat
, buf
->data
));
151 /* Check that we support this format at all */
152 SVN_ERR(check_format(*pformat
));
154 /* Read any options. */
155 SVN_ERR(svn_stream_readline(stream
, &buf
, "\n", &eos
, scratch_pool
));
156 if (!eos
&& strncmp(buf
->data
, "layout sharded ", 15) == 0)
158 /* Check that the argument is numeric. */
159 SVN_ERR(check_format_file_buffer_numeric(buf
->data
, 15, path
,
161 SVN_ERR(svn_cstring_atoi(max_files_per_dir
, buf
->data
+ 15));
164 return svn_error_createf(SVN_ERR_BAD_VERSION_FILE_FORMAT
, NULL
,
165 _("'%s' contains invalid filesystem format option '%s'"),
166 svn_dirent_local_style(path
, scratch_pool
), buf
->data
);
171 /* Write the format number and maximum number of files per directory
172 to a new format file in PATH, possibly expecting to overwrite a
173 previously existing file.
175 Use SCRATCH_POOL for temporary allocation. */
177 svn_fs_x__write_format(svn_fs_t
*fs
,
178 svn_boolean_t overwrite
,
179 apr_pool_t
*scratch_pool
)
182 const char *path
= svn_fs_x__path_format(fs
, scratch_pool
);
183 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
185 SVN_ERR_ASSERT(1 <= ffd
->format
&& ffd
->format
<= SVN_FS_X__FORMAT_NUMBER
);
187 sb
= svn_stringbuf_createf(scratch_pool
, "%d\n", ffd
->format
);
188 svn_stringbuf_appendcstr(sb
, apr_psprintf(scratch_pool
,
189 "layout sharded %d\n",
190 ffd
->max_files_per_dir
));
192 /* svn_io_write_version_file() does a load of magic to allow it to
193 replace version files that already exist. We only need to do
194 that when we're allowed to overwrite an existing file. */
197 /* Create the file */
198 SVN_ERR(svn_io_file_create(path
, sb
->data
, scratch_pool
));
202 SVN_ERR(svn_io_write_atomic2(path
, sb
->data
, sb
->len
,
203 NULL
/* copy_perms_path */,
204 ffd
->flush_to_disk
, scratch_pool
));
207 /* And set the perms to make it read only */
208 return svn_io_set_file_read_only(path
, FALSE
, scratch_pool
);
211 /* Check that BLOCK_SIZE is a valid block / page size, i.e. it is within
212 * the range of what the current system may address in RAM and it is a
213 * power of 2. Assume that the element size within the block is ITEM_SIZE.
214 * Use SCRATCH_POOL for temporary allocations.
217 verify_block_size(apr_int64_t block_size
,
218 apr_size_t item_size
,
220 apr_pool_t
*scratch_pool
)
224 return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE
, NULL
,
225 _("%s is too small for fsfs.conf setting '%s'."),
226 apr_psprintf(scratch_pool
,
231 if (block_size
> SVN_MAX_OBJECT_SIZE
/ item_size
)
232 return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE
, NULL
,
233 _("%s is too large for fsfs.conf setting '%s'."),
234 apr_psprintf(scratch_pool
,
239 /* Ensure it is a power of two.
240 * For positive X, X & (X-1) will reset the lowest bit set.
241 * If the result is 0, at most one bit has been set. */
242 if (0 != (block_size
& (block_size
- 1)))
243 return svn_error_createf(SVN_ERR_BAD_CONFIG_VALUE
, NULL
,
244 _("%s is invalid for fsfs.conf setting '%s' "
245 "because it is not a power of 2."),
246 apr_psprintf(scratch_pool
,
254 /* Read the configuration information of the file system at FS_PATH
255 * and set the respective values in FFD. Use pools as usual.
258 read_config(svn_fs_x__data_t
*ffd
,
260 apr_pool_t
*result_pool
,
261 apr_pool_t
*scratch_pool
)
263 svn_config_t
*config
;
264 apr_int64_t compression_level
;
266 SVN_ERR(svn_config_read3(&config
,
267 svn_dirent_join(fs_path
, PATH_CONFIG
, scratch_pool
),
268 FALSE
, FALSE
, FALSE
, scratch_pool
));
270 /* Initialize ffd->rep_sharing_allowed. */
271 SVN_ERR(svn_config_get_bool(config
, &ffd
->rep_sharing_allowed
,
272 CONFIG_SECTION_REP_SHARING
,
273 CONFIG_OPTION_ENABLE_REP_SHARING
, TRUE
));
275 /* Initialize deltification settings in ffd. */
276 SVN_ERR(svn_config_get_int64(config
, &ffd
->max_deltification_walk
,
277 CONFIG_SECTION_DELTIFICATION
,
278 CONFIG_OPTION_MAX_DELTIFICATION_WALK
,
279 SVN_FS_X_MAX_DELTIFICATION_WALK
));
280 SVN_ERR(svn_config_get_int64(config
, &ffd
->max_linear_deltification
,
281 CONFIG_SECTION_DELTIFICATION
,
282 CONFIG_OPTION_MAX_LINEAR_DELTIFICATION
,
283 SVN_FS_X_MAX_LINEAR_DELTIFICATION
));
284 SVN_ERR(svn_config_get_int64(config
, &compression_level
,
285 CONFIG_SECTION_DELTIFICATION
,
286 CONFIG_OPTION_COMPRESSION_LEVEL
,
287 SVN_DELTA_COMPRESSION_LEVEL_DEFAULT
));
288 ffd
->delta_compression_level
289 = (int)MIN(MAX(SVN_DELTA_COMPRESSION_LEVEL_NONE
, compression_level
),
290 SVN_DELTA_COMPRESSION_LEVEL_MAX
);
292 /* Initialize revprop packing settings in ffd. */
293 SVN_ERR(svn_config_get_bool(config
, &ffd
->compress_packed_revprops
,
294 CONFIG_SECTION_PACKED_REVPROPS
,
295 CONFIG_OPTION_COMPRESS_PACKED_REVPROPS
,
297 SVN_ERR(svn_config_get_int64(config
, &ffd
->revprop_pack_size
,
298 CONFIG_SECTION_PACKED_REVPROPS
,
299 CONFIG_OPTION_REVPROP_PACK_SIZE
,
300 ffd
->compress_packed_revprops
304 ffd
->revprop_pack_size
*= 1024;
306 /* I/O settings in ffd. */
307 SVN_ERR(svn_config_get_int64(config
, &ffd
->block_size
,
309 CONFIG_OPTION_BLOCK_SIZE
,
311 SVN_ERR(svn_config_get_int64(config
, &ffd
->l2p_page_size
,
313 CONFIG_OPTION_L2P_PAGE_SIZE
,
315 SVN_ERR(svn_config_get_int64(config
, &ffd
->p2l_page_size
,
317 CONFIG_OPTION_P2L_PAGE_SIZE
,
320 /* Don't accept unreasonable or illegal values.
321 * Block size and P2L page size are in kbytes;
322 * L2P blocks are arrays of apr_off_t. */
323 SVN_ERR(verify_block_size(ffd
->block_size
, 0x400,
324 CONFIG_OPTION_BLOCK_SIZE
, scratch_pool
));
325 SVN_ERR(verify_block_size(ffd
->p2l_page_size
, 0x400,
326 CONFIG_OPTION_P2L_PAGE_SIZE
, scratch_pool
));
327 SVN_ERR(verify_block_size(ffd
->l2p_page_size
, sizeof(apr_off_t
),
328 CONFIG_OPTION_L2P_PAGE_SIZE
, scratch_pool
));
330 /* convert kBytes to bytes */
331 ffd
->block_size
*= 0x400;
332 ffd
->p2l_page_size
*= 0x400;
333 /* L2P pages are in entries - not in (k)Bytes */
336 SVN_ERR(svn_config_get_bool(config
, &ffd
->pack_after_commit
,
337 CONFIG_SECTION_DEBUG
,
338 CONFIG_OPTION_PACK_AFTER_COMMIT
,
341 /* memcached configuration */
342 SVN_ERR(svn_cache__make_memcache_from_config(&ffd
->memcache
, config
,
343 result_pool
, scratch_pool
));
345 SVN_ERR(svn_config_get_bool(config
, &ffd
->fail_stop
,
346 CONFIG_SECTION_CACHES
, CONFIG_OPTION_FAIL_STOP
,
352 /* Write FS' initial configuration file.
353 * Use SCRATCH_POOL for temporary allocations. */
355 write_config(svn_fs_t
*fs
,
356 apr_pool_t
*scratch_pool
)
358 #define NL APR_EOL_STR
359 static const char * const fsx_conf_contents
=
360 "### This file controls the configuration of the FSX filesystem." NL
362 "[" SVN_CACHE_CONFIG_CATEGORY_MEMCACHED_SERVERS
"]" NL
363 "### These options name memcached servers used to cache internal FSX" NL
364 "### data. See http://www.danga.com/memcached/ for more information on" NL
365 "### memcached. To use memcached with FSX, run one or more memcached" NL
366 "### servers, and specify each of them as an option like so:" NL
367 "# first-server = 127.0.0.1:11211" NL
368 "# remote-memcached = mymemcached.corp.example.com:11212" NL
369 "### The option name is ignored; the value is of the form HOST:PORT." NL
370 "### memcached servers can be shared between multiple repositories;" NL
371 "### however, if you do this, you *must* ensure that repositories have" NL
372 "### distinct UUIDs and paths, or else cached data from one repository" NL
373 "### might be used by another accidentally. Note also that memcached has" NL
374 "### no authentication for reads or writes, so you must ensure that your" NL
375 "### memcached servers are only accessible by trusted users." NL
377 "[" CONFIG_SECTION_CACHES
"]" NL
378 "### When a cache-related error occurs, normally Subversion ignores it" NL
379 "### and continues, logging an error if the server is appropriately" NL
380 "### configured (and ignoring it with file:// access). To make" NL
381 "### Subversion never ignore cache errors, uncomment this line." NL
382 "# " CONFIG_OPTION_FAIL_STOP
" = true" NL
384 "[" CONFIG_SECTION_REP_SHARING
"]" NL
385 "### To conserve space, the filesystem can optionally avoid storing" NL
386 "### duplicate representations. This comes at a slight cost in" NL
387 "### performance, as maintaining a database of shared representations can" NL
388 "### increase commit times. The space savings are dependent upon the size" NL
389 "### of the repository, the number of objects it contains and the amount of" NL
390 "### duplication between them, usually a function of the branching and" NL
391 "### merging process." NL
393 "### The following parameter enables rep-sharing in the repository. It can" NL
394 "### be switched on and off at will, but for best space-saving results" NL
395 "### should be enabled consistently over the life of the repository." NL
396 "### 'svnadmin verify' will check the rep-cache regardless of this setting." NL
397 "### rep-sharing is enabled by default." NL
398 "# " CONFIG_OPTION_ENABLE_REP_SHARING
" = true" NL
400 "[" CONFIG_SECTION_DELTIFICATION
"]" NL
401 "### To conserve space, the filesystem stores data as differences against" NL
402 "### existing representations. This comes at a slight cost in performance," NL
403 "### as calculating differences can increase commit times. Reading data" NL
404 "### will also create higher CPU load and the data will be fragmented." NL
405 "### Since deltification tends to save significant amounts of disk space," NL
406 "### the overall I/O load can actually be lower." NL
408 "### The options in this section allow for tuning the deltification" NL
409 "### strategy. Their effects on data size and server performance may vary" NL
410 "### from one repository to another." NL
412 "### During commit, the server may need to walk the whole change history of" NL
413 "### of a given node to find a suitable deltification base. This linear" NL
414 "### process can impact commit times, svnadmin load and similar operations." NL
415 "### This setting limits the depth of the deltification history. If the" NL
416 "### threshold has been reached, the node will be stored as fulltext and a" NL
417 "### new deltification history begins." NL
418 "### Note, this is unrelated to svn log." NL
419 "### Very large values rarely provide significant additional savings but" NL
420 "### can impact performance greatly - in particular if directory" NL
421 "### deltification has been activated. Very small values may be useful in" NL
422 "### repositories that are dominated by large, changing binaries." NL
423 "### Should be a power of two minus 1. A value of 0 will effectively" NL
424 "### disable deltification." NL
425 "### For 1.9, the default value is 1023." NL
426 "# " CONFIG_OPTION_MAX_DELTIFICATION_WALK
" = 1023" NL
428 "### The skip-delta scheme used by FSX tends to repeatably store redundant" NL
429 "### delta information where a simple delta against the latest version is" NL
430 "### often smaller. By default, 1.9+ will therefore use skip deltas only" NL
431 "### after the linear chain of deltas has grown beyond the threshold" NL
432 "### specified by this setting." NL
433 "### Values up to 64 can result in some reduction in repository size for" NL
434 "### the cost of quickly increasing I/O and CPU costs. Similarly, smaller" NL
435 "### numbers can reduce those costs at the cost of more disk space. For" NL
436 "### rarely read repositories or those containing larger binaries, this may" NL
437 "### present a better trade-off." NL
438 "### Should be a power of two. A value of 1 or smaller will cause the" NL
439 "### exclusive use of skip-deltas." NL
440 "### For 1.8, the default value is 16." NL
441 "# " CONFIG_OPTION_MAX_LINEAR_DELTIFICATION
" = 16" NL
443 "### After deltification, we compress the data through zlib to minimize on-" NL
444 "### disk size. That can be an expensive and ineffective process. This" NL
445 "### setting controls the usage of zlib in future revisions." NL
446 "### Revisions with highly compressible data in them may shrink in size" NL
447 "### if the setting is increased but may take much longer to commit. The" NL
448 "### time taken to uncompress that data again is widely independent of the" NL
449 "### compression level." NL
450 "### Compression will be ineffective if the incoming content is already" NL
451 "### highly compressed. In that case, disabling the compression entirely" NL
452 "### will speed up commits as well as reading the data. Repositories with" NL
453 "### many small compressible files (source code) but also a high percentage" NL
454 "### of large incompressible ones (artwork) may benefit from compression" NL
455 "### levels lowered to e.g. 1." NL
456 "### Valid values are 0 to 9 with 9 providing the highest compression ratio" NL
457 "### and 0 disabling it altogether." NL
458 "### The default value is 5." NL
459 "# " CONFIG_OPTION_COMPRESSION_LEVEL
" = 5" NL
461 "[" CONFIG_SECTION_PACKED_REVPROPS
"]" NL
462 "### This parameter controls the size (in kBytes) of packed revprop files." NL
463 "### Revprops of consecutive revisions will be concatenated into a single" NL
464 "### file up to but not exceeding the threshold given here. However, each" NL
465 "### pack file may be much smaller and revprops of a single revision may be" NL
466 "### much larger than the limit set here. The threshold will be applied" NL
467 "### before optional compression takes place." NL
468 "### Large values will reduce disk space usage at the expense of increased" NL
469 "### latency and CPU usage reading and changing individual revprops. They" NL
470 "### become an advantage when revprop caching has been enabled because a" NL
471 "### lot of data can be read in one go. Values smaller than 4 kByte will" NL
472 "### not improve latency any further and quickly render revprop packing" NL
473 "### ineffective." NL
474 "### revprop-pack-size is 64 kBytes by default for non-compressed revprop" NL
475 "### pack files and 256 kBytes when compression has been enabled." NL
476 "# " CONFIG_OPTION_REVPROP_PACK_SIZE
" = 64" NL
478 "### To save disk space, packed revprop files may be compressed. Standard" NL
479 "### revprops tend to allow for very effective compression. Reading and" NL
480 "### even more so writing, become significantly more CPU intensive. With" NL
481 "### revprop caching enabled, the overhead can be offset by reduced I/O" NL
482 "### unless you often modify revprops after packing." NL
483 "### Compressing packed revprops is enabled by default." NL
484 "# " CONFIG_OPTION_COMPRESS_PACKED_REVPROPS
" = true" NL
486 "[" CONFIG_SECTION_IO
"]" NL
487 "### Parameters in this section control the data access granularity in" NL
488 "### format 7 repositories and later. The defaults should translate into" NL
489 "### decent performance over a wide range of setups." NL
491 "### When a specific piece of information needs to be read from disk, a" NL
492 "### data block is being read at once and its contents are being cached." NL
493 "### If the repository is being stored on a RAID, the block size should be" NL
494 "### either 50% or 100% of RAID block size / granularity. Also, your file" NL
495 "### system blocks/clusters should be properly aligned and sized. In that" NL
496 "### setup, each access will hit only one disk (minimizes I/O load) but" NL
497 "### uses all the data provided by the disk in a single access." NL
498 "### For SSD-based storage systems, slightly lower values around 16 kB" NL
499 "### may improve latency while still maximizing throughput." NL
500 "### Can be changed at any time but must be a power of 2." NL
501 "### block-size is given in kBytes and with a default of 64 kBytes." NL
502 "# " CONFIG_OPTION_BLOCK_SIZE
" = 64" NL
504 "### The log-to-phys index maps data item numbers to offsets within the" NL
505 "### rev or pack file. This index is organized in pages of a fixed maximum" NL
506 "### capacity. To access an item, the page table and the respective page" NL
507 "### must be read." NL
508 "### This parameter only affects revisions with thousands of changed paths." NL
509 "### If you have several extremely large revisions (~1 mio changes), think" NL
510 "### about increasing this setting. Reducing the value will rarely result" NL
511 "### in a net speedup." NL
512 "### This is an expert setting. Must be a power of 2." NL
513 "### l2p-page-size is 8192 entries by default." NL
514 "# " CONFIG_OPTION_L2P_PAGE_SIZE
" = 8192" NL
516 "### The phys-to-log index maps positions within the rev or pack file to" NL
517 "### to data items, i.e. describes what piece of information is being" NL
518 "### stored at any particular offset. The index describes the rev file" NL
519 "### in chunks (pages) and keeps a global list of all those pages. Large" NL
520 "### pages mean a shorter page table but a larger per-page description of" NL
521 "### data items in it. The latency sweet spot depends on the change size" NL
522 "### distribution but covers a relatively wide range." NL
523 "### If the repository contains very large files, i.e. individual changes" NL
524 "### of tens of MB each, increasing the page size will shorten the index" NL
525 "### file at the expense of a slightly increased latency in sections with" NL
526 "### smaller changes." NL
527 "### For source code repositories, this should be about 16x the block-size." NL
528 "### Must be a power of 2." NL
529 "### p2l-page-size is given in kBytes and with a default of 1024 kBytes." NL
530 "# " CONFIG_OPTION_P2L_PAGE_SIZE
" = 1024" NL
533 return svn_io_file_create(svn_dirent_join(fs
->path
, PATH_CONFIG
,
535 fsx_conf_contents
, scratch_pool
);
538 /* Read / Evaluate the global configuration in FS->CONFIG to set up
539 * parameters in FS. */
541 read_global_config(svn_fs_t
*fs
)
543 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
545 ffd
->flush_to_disk
= !svn_hash__get_bool(fs
->config
,
546 SVN_FS_CONFIG_NO_FLUSH_TO_DISK
,
552 /* Read FS's UUID file and store the data in the FS struct. */
554 read_uuid(svn_fs_t
*fs
,
555 apr_pool_t
*scratch_pool
)
557 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
558 apr_file_t
*uuid_file
;
559 char buf
[APR_UUID_FORMATTED_LENGTH
+ 2];
562 /* Read the repository uuid. */
563 SVN_ERR(svn_io_file_open(&uuid_file
, svn_fs_x__path_uuid(fs
, scratch_pool
),
564 APR_READ
| APR_BUFFERED
, APR_OS_DEFAULT
,
568 SVN_ERR(svn_io_read_length_line(uuid_file
, buf
, &limit
, scratch_pool
));
569 fs
->uuid
= apr_pstrdup(fs
->pool
, buf
);
571 /* Read the instance ID. */
573 SVN_ERR(svn_io_read_length_line(uuid_file
, buf
, &limit
,
575 ffd
->instance_id
= apr_pstrdup(fs
->pool
, buf
);
577 SVN_ERR(svn_io_file_close(uuid_file
, scratch_pool
));
583 svn_fs_x__read_format_file(svn_fs_t
*fs
,
584 apr_pool_t
*scratch_pool
)
586 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
587 int format
, max_files_per_dir
;
589 /* Read info from format file. */
590 SVN_ERR(read_format(&format
, &max_files_per_dir
,
591 svn_fs_x__path_format(fs
, scratch_pool
), scratch_pool
));
593 /* Now that we've got *all* info, store / update values in FFD. */
594 ffd
->format
= format
;
595 ffd
->max_files_per_dir
= max_files_per_dir
;
601 svn_fs_x__open(svn_fs_t
*fs
,
603 apr_pool_t
*scratch_pool
)
605 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
606 fs
->path
= apr_pstrdup(fs
->pool
, path
);
608 /* Read the FS format file. */
609 SVN_ERR(svn_fs_x__read_format_file(fs
, scratch_pool
));
611 /* Read in and cache the repository uuid. */
612 SVN_ERR(read_uuid(fs
, scratch_pool
));
614 /* Read the min unpacked revision. */
615 SVN_ERR(svn_fs_x__update_min_unpacked_rev(fs
, scratch_pool
));
617 /* Read the configuration file. */
618 SVN_ERR(read_config(ffd
, fs
->path
, fs
->pool
, scratch_pool
));
620 /* Global configuration options. */
621 SVN_ERR(read_global_config(fs
));
623 ffd
->youngest_rev_cache
= 0;
628 /* Baton type bridging svn_fs_x__upgrade and upgrade_body carrying
629 * parameters over between them. */
630 typedef struct upgrade_baton_t
633 svn_fs_upgrade_notify_t notify_func
;
635 svn_cancel_func_t cancel_func
;
639 /* Upgrade the FS given in upgrade_baton_t *)BATON to the latest format
640 * version. Apply options an invoke callback from that BATON.
641 * Temporary allocations are to be made from SCRATCH_POOL.
643 * At the moment, this is a simple placeholder as we don't support upgrades
644 * from experimental FSX versions.
647 upgrade_body(void *baton
,
648 apr_pool_t
*scratch_pool
)
650 upgrade_baton_t
*upgrade_baton
= baton
;
651 svn_fs_t
*fs
= upgrade_baton
->fs
;
652 int format
, max_files_per_dir
;
653 const char *format_path
= svn_fs_x__path_format(fs
, scratch_pool
);
655 /* Read the FS format number and max-files-per-dir setting. */
656 SVN_ERR(read_format(&format
, &max_files_per_dir
, format_path
,
659 /* If we're already up-to-date, there's nothing else to be done here. */
660 if (format
== SVN_FS_X__FORMAT_NUMBER
)
669 svn_fs_x__upgrade(svn_fs_t
*fs
,
670 svn_fs_upgrade_notify_t notify_func
,
672 svn_cancel_func_t cancel_func
,
674 apr_pool_t
*scratch_pool
)
676 upgrade_baton_t baton
;
678 baton
.notify_func
= notify_func
;
679 baton
.notify_baton
= notify_baton
;
680 baton
.cancel_func
= cancel_func
;
681 baton
.cancel_baton
= cancel_baton
;
683 return svn_fs_x__with_all_locks(fs
, upgrade_body
, (void *)&baton
,
689 svn_fs_x__youngest_rev(svn_revnum_t
*youngest_p
,
691 apr_pool_t
*scratch_pool
)
693 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
694 SVN_ERR(svn_fs_x__read_current(youngest_p
, fs
, scratch_pool
));
695 ffd
->youngest_rev_cache
= *youngest_p
;
701 svn_fs_x__ensure_revision_exists(svn_revnum_t rev
,
703 apr_pool_t
*scratch_pool
)
705 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
707 if (! SVN_IS_VALID_REVNUM(rev
))
708 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION
, NULL
,
709 _("Invalid revision number '%ld'"), rev
);
712 /* Did the revision exist the last time we checked the current
714 if (rev
<= ffd
->youngest_rev_cache
)
717 SVN_ERR(svn_fs_x__read_current(&ffd
->youngest_rev_cache
, fs
, scratch_pool
));
720 if (rev
<= ffd
->youngest_rev_cache
)
723 return svn_error_createf(SVN_ERR_FS_NO_SUCH_REVISION
, NULL
,
724 _("No such revision %ld"), rev
);
729 svn_fs_x__file_length(svn_filesize_t
*length
,
730 svn_fs_x__noderev_t
*noderev
)
732 if (noderev
->data_rep
)
733 *length
= noderev
->data_rep
->expanded_size
;
741 svn_fs_x__file_text_rep_equal(svn_fs_x__representation_t
*a
,
742 svn_fs_x__representation_t
*b
)
744 svn_boolean_t a_empty
= a
== NULL
|| a
->expanded_size
== 0;
745 svn_boolean_t b_empty
= b
== NULL
|| b
->expanded_size
== 0;
747 /* This makes sure that neither rep will be NULL later on */
748 if (a_empty
&& b_empty
)
751 if (a_empty
!= b_empty
)
754 /* Same physical representation? Note that these IDs are always up-to-date
755 instead of e.g. being set lazily. */
756 if (svn_fs_x__id_eq(&a
->id
, &b
->id
))
759 /* Contents are equal if the checksums match. These are also always known.
761 return memcmp(a
->md5_digest
, b
->md5_digest
, sizeof(a
->md5_digest
)) == 0
762 && memcmp(a
->sha1_digest
, b
->sha1_digest
, sizeof(a
->sha1_digest
)) == 0;
766 svn_fs_x__prop_rep_equal(svn_boolean_t
*equal
,
768 svn_fs_x__noderev_t
*a
,
769 svn_fs_x__noderev_t
*b
,
770 svn_boolean_t strict
,
771 apr_pool_t
*scratch_pool
)
773 svn_fs_x__representation_t
*rep_a
= a
->prop_rep
;
774 svn_fs_x__representation_t
*rep_b
= b
->prop_rep
;
775 apr_hash_t
*proplist_a
;
776 apr_hash_t
*proplist_b
;
778 /* Mainly for a==b==NULL */
785 /* Committed property lists can be compared quickly */
787 && svn_fs_x__is_revision(rep_a
->id
.change_set
)
788 && svn_fs_x__is_revision(rep_b
->id
.change_set
))
790 /* MD5 must be given. Having the same checksum is good enough for
791 accepting the prop lists as equal. */
792 *equal
= memcmp(rep_a
->md5_digest
, rep_b
->md5_digest
,
793 sizeof(rep_a
->md5_digest
)) == 0;
797 /* Same path in same txn? */
798 if (svn_fs_x__id_eq(&a
->noderev_id
, &b
->noderev_id
))
804 /* Skip the expensive bits unless we are in strict mode.
805 Simply assume that there is a different. */
812 /* At least one of the reps has been modified in a txn.
813 Fetch and compare them. */
814 SVN_ERR(svn_fs_x__get_proplist(&proplist_a
, fs
, a
, scratch_pool
,
816 SVN_ERR(svn_fs_x__get_proplist(&proplist_b
, fs
, b
, scratch_pool
,
819 *equal
= svn_fs__prop_lists_equal(proplist_a
, proplist_b
, scratch_pool
);
825 svn_fs_x__file_checksum(svn_checksum_t
**checksum
,
826 svn_fs_x__noderev_t
*noderev
,
827 svn_checksum_kind_t kind
,
828 apr_pool_t
*result_pool
)
832 if (noderev
->data_rep
)
839 case svn_checksum_md5
:
840 temp
.digest
= noderev
->data_rep
->md5_digest
;
843 case svn_checksum_sha1
:
844 if (! noderev
->data_rep
->has_sha1
)
847 temp
.digest
= noderev
->data_rep
->sha1_digest
;
854 *checksum
= svn_checksum_dup(&temp
, result_pool
);
860 svn_fs_x__representation_t
*
861 svn_fs_x__rep_copy(svn_fs_x__representation_t
*rep
,
862 apr_pool_t
*result_pool
)
867 return apr_pmemdup(result_pool
, rep
, sizeof(*rep
));
871 /* Write out the zeroth revision for filesystem FS.
872 Perform temporary allocations in SCRATCH_POOL. */
874 write_revision_zero(svn_fs_t
*fs
,
875 apr_pool_t
*scratch_pool
)
877 const char *path_revision_zero
= svn_fs_x__path_rev(fs
, 0, scratch_pool
);
878 apr_hash_t
*proplist
;
881 apr_array_header_t
*index_entries
;
882 svn_fs_x__p2l_entry_t
*entry
;
883 svn_fs_x__revision_file_t
*rev_file
;
884 apr_file_t
*apr_file
;
885 const char *l2p_proto_index
, *p2l_proto_index
;
887 /* Construct a skeleton r0 with no indexes. */
888 svn_string_t
*noderev_str
= svn_string_create("id: 2+0\n"
896 svn_string_t
*changes_str
= svn_string_create("\n",
898 svn_string_t
*r0
= svn_string_createf(scratch_pool
, "%s%s",
902 /* Write skeleton r0 to disk. */
903 SVN_ERR(svn_io_file_create(path_revision_zero
, r0
->data
, scratch_pool
));
905 /* Construct the index P2L contents: describe the 2 items we have.
906 Be sure to create them in on-disk order. */
907 index_entries
= apr_array_make(scratch_pool
, 2, sizeof(entry
));
909 entry
= apr_pcalloc(scratch_pool
, sizeof(*entry
));
911 entry
->size
= (apr_off_t
)noderev_str
->len
;
912 entry
->type
= SVN_FS_X__ITEM_TYPE_NODEREV
;
913 entry
->item_count
= 1;
914 entry
->items
= apr_pcalloc(scratch_pool
, sizeof(*entry
->items
));
915 entry
->items
[0].change_set
= 0;
916 entry
->items
[0].number
= SVN_FS_X__ITEM_INDEX_ROOT_NODE
;
917 APR_ARRAY_PUSH(index_entries
, svn_fs_x__p2l_entry_t
*) = entry
;
919 entry
= apr_pcalloc(scratch_pool
, sizeof(*entry
));
920 entry
->offset
= (apr_off_t
)noderev_str
->len
;
921 entry
->size
= (apr_off_t
)changes_str
->len
;
922 entry
->type
= SVN_FS_X__ITEM_TYPE_CHANGES
;
923 entry
->item_count
= 1;
924 entry
->items
= apr_pcalloc(scratch_pool
, sizeof(*entry
->items
));
925 entry
->items
[0].change_set
= 0;
926 entry
->items
[0].number
= SVN_FS_X__ITEM_INDEX_CHANGES
;
927 APR_ARRAY_PUSH(index_entries
, svn_fs_x__p2l_entry_t
*) = entry
;
929 /* Now re-open r0, create proto-index files from our entries and
930 rewrite the index section of r0. */
931 SVN_ERR(svn_fs_x__rev_file_open_writable(&rev_file
, fs
, 0,
932 scratch_pool
, scratch_pool
));
933 SVN_ERR(svn_fs_x__p2l_index_from_p2l_entries(&p2l_proto_index
, fs
,
934 rev_file
, index_entries
,
935 scratch_pool
, scratch_pool
));
936 SVN_ERR(svn_fs_x__l2p_index_from_p2l_entries(&l2p_proto_index
, fs
,
938 scratch_pool
, scratch_pool
));
939 SVN_ERR(svn_fs_x__rev_file_get(&apr_file
, rev_file
));
940 SVN_ERR(svn_fs_x__add_index_data(fs
, apr_file
, l2p_proto_index
,
941 p2l_proto_index
, 0, scratch_pool
));
942 SVN_ERR(svn_fs_x__close_revision_file(rev_file
));
944 SVN_ERR(svn_io_set_file_read_only(path_revision_zero
, FALSE
, scratch_pool
));
946 /* Set a date on revision 0. */
947 date
.data
= svn_time_to_cstring(apr_time_now(), scratch_pool
);
948 date
.len
= strlen(date
.data
);
949 proplist
= apr_hash_make(scratch_pool
);
950 svn_hash_sets(proplist
, SVN_PROP_REVISION_DATE
, &date
);
952 SVN_ERR(svn_io_file_open(&apr_file
,
953 svn_fs_x__path_revprops(fs
, 0, scratch_pool
),
954 APR_WRITE
| APR_CREATE
, APR_OS_DEFAULT
,
956 SVN_ERR(svn_fs_x__write_non_packed_revprops(apr_file
, proplist
,
958 SVN_ERR(svn_io_file_close(apr_file
, scratch_pool
));
964 svn_fs_x__create_file_tree(svn_fs_t
*fs
,
968 apr_pool_t
*scratch_pool
)
970 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
972 fs
->path
= apr_pstrdup(fs
->pool
, path
);
973 ffd
->format
= format
;
975 /* Use an appropriate sharding mode if supported by the format. */
976 ffd
->max_files_per_dir
= shard_size
;
978 /* Create the revision data directories. */
979 SVN_ERR(svn_io_make_dir_recursively(
980 svn_fs_x__path_shard(fs
, 0, scratch_pool
),
983 /* Create the transaction directory. */
984 SVN_ERR(svn_io_make_dir_recursively(
985 svn_fs_x__path_txns_dir(fs
, scratch_pool
),
988 /* Create the protorevs directory. */
989 SVN_ERR(svn_io_make_dir_recursively(
990 svn_fs_x__path_txn_proto_revs(fs
, scratch_pool
),
993 /* Create the 'current' file. */
994 SVN_ERR(svn_io_file_create(svn_fs_x__path_current(fs
, scratch_pool
),
995 "0\n", scratch_pool
));
997 /* Create the 'uuid' file. */
998 SVN_ERR(svn_io_file_create_empty(svn_fs_x__path_lock(fs
, scratch_pool
),
1000 SVN_ERR(svn_fs_x__set_uuid(fs
, NULL
, NULL
, FALSE
, scratch_pool
));
1002 /* Create the fsfs.conf file. */
1003 SVN_ERR(write_config(fs
, scratch_pool
));
1004 SVN_ERR(read_config(ffd
, fs
->path
, fs
->pool
, scratch_pool
));
1006 /* Global configuration options. */
1007 SVN_ERR(read_global_config(fs
));
1009 /* Add revision 0. */
1010 SVN_ERR(write_revision_zero(fs
, scratch_pool
));
1012 /* Create the min unpacked rev file. */
1013 SVN_ERR(svn_io_file_create(
1014 svn_fs_x__path_min_unpacked_rev(fs
, scratch_pool
),
1015 "0\n", scratch_pool
));
1017 /* Create the txn-current file if the repository supports
1018 the transaction sequence file. */
1019 SVN_ERR(svn_io_file_create(svn_fs_x__path_txn_current(fs
, scratch_pool
),
1020 "0\n", scratch_pool
));
1021 SVN_ERR(svn_io_file_create_empty(
1022 svn_fs_x__path_txn_current_lock(fs
, scratch_pool
),
1025 /* Initialize the revprop caching info. */
1026 SVN_ERR(svn_io_file_create_empty(
1027 svn_fs_x__path_revprop_generation(fs
, scratch_pool
),
1029 SVN_ERR(svn_fs_x__reset_revprop_generation_file(fs
, scratch_pool
));
1031 ffd
->youngest_rev_cache
= 0;
1032 return SVN_NO_ERROR
;
1036 svn_fs_x__create(svn_fs_t
*fs
,
1038 apr_pool_t
*scratch_pool
)
1040 int format
= SVN_FS_X__FORMAT_NUMBER
;
1041 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
1043 fs
->path
= apr_pstrdup(fs
->pool
, path
);
1044 /* See if compatibility with older versions was explicitly requested. */
1047 svn_version_t
*compatible_version
;
1048 SVN_ERR(svn_fs__compatible_version(&compatible_version
, fs
->config
,
1051 /* select format number */
1052 switch(compatible_version
->minor
)
1062 case 8: return svn_error_create(SVN_ERR_FS_UNSUPPORTED_FORMAT
, NULL
,
1063 _("FSX is not compatible with Subversion prior to 1.9"));
1065 default:format
= SVN_FS_X__FORMAT_NUMBER
;
1069 /* Actual FS creation. */
1070 SVN_ERR(svn_fs_x__create_file_tree(fs
, path
, format
,
1071 SVN_FS_X_DEFAULT_MAX_FILES_PER_DIR
,
1074 /* This filesystem is ready. Stamp it with a format number. */
1075 SVN_ERR(svn_fs_x__write_format(fs
, FALSE
, scratch_pool
));
1077 ffd
->youngest_rev_cache
= 0;
1078 return SVN_NO_ERROR
;
1082 svn_fs_x__set_uuid(svn_fs_t
*fs
,
1084 const char *instance_id
,
1085 svn_boolean_t overwrite
,
1086 apr_pool_t
*scratch_pool
)
1088 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
1089 const char *uuid_path
= svn_fs_x__path_uuid(fs
, scratch_pool
);
1090 svn_stringbuf_t
*contents
= svn_stringbuf_create_empty(scratch_pool
);
1093 uuid
= svn_uuid_generate(scratch_pool
);
1096 instance_id
= svn_uuid_generate(scratch_pool
);
1098 svn_stringbuf_appendcstr(contents
, uuid
);
1099 svn_stringbuf_appendcstr(contents
, "\n");
1100 svn_stringbuf_appendcstr(contents
, instance_id
);
1101 svn_stringbuf_appendcstr(contents
, "\n");
1103 /* We use the permissions of the 'current' file, because the 'uuid'
1104 file does not exist during repository creation.
1106 svn_io_write_atomic2() does a load of magic to allow it to
1107 replace version files that already exist. We only need to do
1108 that when we're allowed to overwrite an existing file. */
1111 /* Create the file */
1112 SVN_ERR(svn_io_file_create(uuid_path
, contents
->data
, scratch_pool
));
1116 SVN_ERR(svn_io_write_atomic2(uuid_path
, contents
->data
, contents
->len
,
1118 svn_fs_x__path_current(fs
, scratch_pool
),
1119 ffd
->flush_to_disk
, scratch_pool
));
1122 fs
->uuid
= apr_pstrdup(fs
->pool
, uuid
);
1123 ffd
->instance_id
= apr_pstrdup(fs
->pool
, instance_id
);
1125 return SVN_NO_ERROR
;
1128 /** Node origin lazy cache. */
1130 /* If directory PATH does not exist, create it and give it the same
1131 permissions as FS_path.*/
1133 svn_fs_x__ensure_dir_exists(const char *path
,
1134 const char *fs_path
,
1135 apr_pool_t
*scratch_pool
)
1137 svn_error_t
*err
= svn_io_dir_make(path
, APR_OS_DEFAULT
, scratch_pool
);
1138 if (err
&& APR_STATUS_IS_EEXIST(err
->apr_err
))
1140 svn_error_clear(err
);
1141 return SVN_NO_ERROR
;
1145 /* We successfully created a new directory. Dup the permissions
1147 return svn_io_copy_perms(fs_path
, path
, scratch_pool
);
1154 svn_fs_x__revision_prop(svn_string_t
**value_p
,
1157 const char *propname
,
1158 svn_boolean_t refresh
,
1159 apr_pool_t
*result_pool
,
1160 apr_pool_t
*scratch_pool
)
1164 SVN_ERR(svn_fs__check_fs(fs
, TRUE
));
1165 SVN_ERR(svn_fs_x__get_revision_proplist(&table
, fs
, rev
, FALSE
, refresh
,
1166 scratch_pool
, scratch_pool
));
1168 *value_p
= svn_string_dup(svn_hash_gets(table
, propname
), result_pool
);
1170 return SVN_NO_ERROR
;
1174 /* Baton used for change_rev_prop_body below. */
1175 typedef struct change_rev_prop_baton_t
{
1179 const svn_string_t
*const *old_value_p
;
1180 const svn_string_t
*value
;
1181 } change_rev_prop_baton_t
;
1183 /* The work-horse for svn_fs_x__change_rev_prop, called with the FS
1184 write lock. This implements the svn_fs_x__with_write_lock()
1185 'body' callback type. BATON is a 'change_rev_prop_baton_t *'. */
1186 static svn_error_t
*
1187 change_rev_prop_body(void *baton
,
1188 apr_pool_t
*scratch_pool
)
1190 change_rev_prop_baton_t
*cb
= baton
;
1192 const svn_string_t
*present_value
;
1194 /* Read current revprop values from disk (never from cache).
1195 Even if somehow the cache got out of sync, we want to make sure that
1196 we read, update and write up-to-date data. */
1197 SVN_ERR(svn_fs_x__get_revision_proplist(&table
, cb
->fs
, cb
->rev
, TRUE
,
1198 TRUE
, scratch_pool
, scratch_pool
));
1199 present_value
= svn_hash_gets(table
, cb
->name
);
1201 if (cb
->old_value_p
)
1203 const svn_string_t
*wanted_value
= *cb
->old_value_p
;
1204 if ((!wanted_value
!= !present_value
)
1205 || (wanted_value
&& present_value
1206 && !svn_string_compare(wanted_value
, present_value
)))
1208 /* What we expected isn't what we found. */
1209 return svn_error_createf(SVN_ERR_FS_PROP_BASEVALUE_MISMATCH
, NULL
,
1210 _("revprop '%s' has unexpected value in "
1217 /* If the prop-set is a no-op, skip the actual write. */
1218 if ((!present_value
&& !cb
->value
)
1219 || (present_value
&& cb
->value
1220 && svn_string_compare(present_value
, cb
->value
)))
1221 return SVN_NO_ERROR
;
1223 svn_hash_sets(table
, cb
->name
, cb
->value
);
1225 return svn_fs_x__set_revision_proplist(cb
->fs
, cb
->rev
, table
,
1230 svn_fs_x__change_rev_prop(svn_fs_t
*fs
,
1233 const svn_string_t
*const *old_value_p
,
1234 const svn_string_t
*value
,
1235 apr_pool_t
*scratch_pool
)
1237 change_rev_prop_baton_t cb
;
1239 SVN_ERR(svn_fs__check_fs(fs
, TRUE
));
1244 cb
.old_value_p
= old_value_p
;
1247 return svn_fs_x__with_write_lock(fs
, change_rev_prop_body
, &cb
,
1253 svn_fs_x__info_format(int *fs_format
,
1254 svn_version_t
**supports_version
,
1256 apr_pool_t
*result_pool
,
1257 apr_pool_t
*scratch_pool
)
1259 svn_fs_x__data_t
*ffd
= fs
->fsap_data
;
1260 *fs_format
= ffd
->format
;
1261 *supports_version
= apr_palloc(result_pool
, sizeof(svn_version_t
));
1263 (*supports_version
)->major
= SVN_VER_MAJOR
;
1264 (*supports_version
)->minor
= 9;
1265 (*supports_version
)->patch
= 0;
1266 (*supports_version
)->tag
= "";
1268 switch (ffd
->format
)
1273 (*supports_version
)->minor
= 10;
1276 # if SVN_FS_X__FORMAT_NUMBER != 2
1277 # error "Need to add a 'case' statement here"
1282 return SVN_NO_ERROR
;
1286 svn_fs_x__info_config_files(apr_array_header_t
**files
,
1288 apr_pool_t
*result_pool
,
1289 apr_pool_t
*scratch_pool
)
1291 *files
= apr_array_make(result_pool
, 1, sizeof(const char *));
1292 APR_ARRAY_PUSH(*files
, const char *) = svn_dirent_join(fs
->path
, PATH_CONFIG
,
1294 return SVN_NO_ERROR
;