?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
avoid needless use of STAILQ_FOREACH_SAFE, we are not removing elements here
[got-portable.git] / lib / repository.c
blobc7b9dbea6bfeb4a26988cf2367b2492119b8242f
1 /*
2 * Copyright (c) 2018, 2019, 2020 Stefan Sperling <stsp@openbsd.org>
3 *
4 * Permission to use, copy, modify, and distribute this software for any
5 * purpose with or without fee is hereby granted, provided that the above
6 * copyright notice and this permission notice appear in all copies.
7 *
8 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
9 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
10 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
11 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
12 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
13 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
14 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
15 */
16
17 #include "got_compat.h"
18
19 #include <sys/types.h>
20 #include <sys/queue.h>
21 #include <sys/uio.h>
22 #include <sys/socket.h>
23 #include <sys/stat.h>
24 #include <sys/mman.h>
25 #include <sys/resource.h>
26
27 #include <ctype.h>
28 #include <fcntl.h>
29 #include <fnmatch.h>
30 #include <limits.h>
31 #include <dirent.h>
32 #include <stdlib.h>
33 #include <stdio.h>
34 #include <string.h>
35 #include <time.h>
36 #include <unistd.h>
37 #include <zlib.h>
38 #include <errno.h>
39 #include <libgen.h>
40 #include <stdint.h>
41
42 #include "bloom.h"
43
44 #include "got_error.h"
45 #include "got_reference.h"
46 #include "got_repository.h"
47 #include "got_path.h"
48 #include "got_cancel.h"
49 #include "got_object.h"
50 #include "got_opentemp.h"
51
52 #include "got_lib_delta.h"
53 #include "got_lib_delta_cache.h"
54 #include "got_lib_hash.h"
55 #include "got_lib_inflate.h"
56 #include "got_lib_object.h"
57 #include "got_lib_object_parse.h"
58 #include "got_lib_object_create.h"
59 #include "got_lib_pack.h"
60 #include "got_lib_privsep.h"
61 #include "got_lib_object_cache.h"
62 #include "got_lib_repository.h"
63 #include "got_lib_gotconfig.h"
64
65 #ifndef nitems
66 #define nitems(_a) (sizeof(_a) / sizeof((_a)[0]))
67 #endif
68
69 #define GOT_PACK_NUM_TEMPFILES GOT_PACK_CACHE_SIZE * 2
70
71 RB_PROTOTYPE(got_packidx_bloom_filter_tree, got_packidx_bloom_filter, entry,
72 got_packidx_bloom_filter_cmp);
73
74 static inline int
75 is_boolean_val(const char *val)
76 {
77 return (strcasecmp(val, "true") == 0 ||
78 strcasecmp(val, "false") == 0 ||
79 strcasecmp(val, "on") == 0 ||
80 strcasecmp(val, "off") == 0 ||
81 strcasecmp(val, "yes") == 0 ||
82 strcasecmp(val, "no") == 0 ||
83 strcasecmp(val, "1") == 0 ||
84 strcasecmp(val, "0") == 0);
85 }
86
87 static inline int
88 get_boolean_val(const char *val)
89 {
90 return (strcasecmp(val, "true") == 0 ||
91 strcasecmp(val, "on") == 0 ||
92 strcasecmp(val, "yes") == 0 ||
93 strcasecmp(val, "1") == 0);
94 }
95
96 const char *
97 got_repo_get_path(struct got_repository *repo)
98 {
99 return repo->path;
100 }
101
102 const char *
103 got_repo_get_path_git_dir(struct got_repository *repo)
104 {
105 return repo->path_git_dir;
106 }
107
108 int
109 got_repo_get_fd(struct got_repository *repo)
110 {
111 return repo->gitdir_fd;
112 }
113
114 enum got_hash_algorithm
115 got_repo_get_object_format(struct got_repository *repo)
116 {
117 return repo->algo;
118 }
119
120 const char *
121 got_repo_get_gitconfig_author_name(struct got_repository *repo)
122 {
123 return repo->gitconfig_author_name;
124 }
125
126 const char *
127 got_repo_get_gitconfig_author_email(struct got_repository *repo)
128 {
129 return repo->gitconfig_author_email;
130 }
131
132 const char *
133 got_repo_get_global_gitconfig_author_name(struct got_repository *repo)
134 {
135 return repo->global_gitconfig_author_name;
136 }
137
138 const char *
139 got_repo_get_global_gitconfig_author_email(struct got_repository *repo)
140 {
141 return repo->global_gitconfig_author_email;
142 }
143
144 const char *
145 got_repo_get_gitconfig_owner(struct got_repository *repo)
146 {
147 return repo->gitconfig_owner;
148 }
149
150 int
151 got_repo_has_extension(struct got_repository *repo, const char *ext)
152 {
153 int i;
154
155 for (i = 0; i < repo->nextensions; ++i) {
156 if (!strcasecmp(ext, repo->extnames[i]))
157 return get_boolean_val(repo->extvals[i]);
158 }
159
160 return 0;
161 }
162
163 int
164 got_repo_is_bare(struct got_repository *repo)
165 {
166 return (strcmp(repo->path, repo->path_git_dir) == 0);
167 }
168
169 static char *
170 get_path_git_child(struct got_repository *repo, const char *basename)
171 {
172 char *path_child;
173
174 if (asprintf(&path_child, "%s/%s", repo->path_git_dir,
175 basename) == -1)
176 return NULL;
177
178 return path_child;
179 }
180
181 char *
182 got_repo_get_path_objects(struct got_repository *repo)
183 {
184 return get_path_git_child(repo, GOT_OBJECTS_DIR);
185 }
186
187 char *
188 got_repo_get_path_objects_pack(struct got_repository *repo)
189 {
190 return get_path_git_child(repo, GOT_OBJECTS_PACK_DIR);
191 }
192
193 char *
194 got_repo_get_path_refs(struct got_repository *repo)
195 {
196 return get_path_git_child(repo, GOT_REFS_DIR);
197 }
198
199 char *
200 got_repo_get_path_packed_refs(struct got_repository *repo)
201 {
202 return get_path_git_child(repo, GOT_PACKED_REFS_FILE);
203 }
204
205 static char *
206 get_path_head(struct got_repository *repo)
207 {
208 return get_path_git_child(repo, GOT_HEAD_FILE);
209 }
210
211 char *
212 got_repo_get_path_gitconfig(struct got_repository *repo)
213 {
214 return get_path_git_child(repo, GOT_GITCONFIG);
215 }
216
217 char *
218 got_repo_get_path_gotconfig(struct got_repository *repo)
219 {
220 return get_path_git_child(repo, GOT_GOTCONFIG_FILENAME);
221 }
222
223 const struct got_gotconfig *
224 got_repo_get_gotconfig(struct got_repository *repo)
225 {
226 return repo->gotconfig;
227 }
228
229 void
230 got_repo_get_gitconfig_remotes(int *nremotes,
231 const struct got_remote_repo **remotes, struct got_repository *repo)
232 {
233 *nremotes = repo->ngitconfig_remotes;
234 *remotes = repo->gitconfig_remotes;
235 }
236
237 static int
238 is_git_repo(struct got_repository *repo)
239 {
240 const char *path_git = got_repo_get_path_git_dir(repo);
241 char *path_objects = got_repo_get_path_objects(repo);
242 char *path_refs = got_repo_get_path_refs(repo);
243 char *path_head = get_path_head(repo);
244 int ret = 0;
245 struct stat sb;
246 struct got_reference *head_ref;
247
248 if (lstat(path_git, &sb) == -1)
249 goto done;
250 if (!S_ISDIR(sb.st_mode))
251 goto done;
252
253 if (lstat(path_objects, &sb) == -1)
254 goto done;
255 if (!S_ISDIR(sb.st_mode))
256 goto done;
257
258 if (lstat(path_refs, &sb) == -1)
259 goto done;
260 if (!S_ISDIR(sb.st_mode))
261 goto done;
262
263 if (lstat(path_head, &sb) == -1)
264 goto done;
265 if (!S_ISREG(sb.st_mode))
266 goto done;
267
268 /* Check if the HEAD reference can be opened. */
269 if (got_ref_open(&head_ref, repo, GOT_REF_HEAD, 0) != NULL)
270 goto done;
271 got_ref_close(head_ref);
272
273 ret = 1;
274 done:
275 free(path_objects);
276 free(path_refs);
277 free(path_head);
278 return ret;
279
280 }
281
282 static const struct got_error *
283 close_tempfiles(int *fds, size_t nfds)
284 {
285 const struct got_error *err = NULL;
286 int i;
287
288 for (i = 0; i < nfds; i++) {
289 if (fds[i] == -1)
290 continue;
291 if (close(fds[i]) == -1) {
292 err = got_error_from_errno("close");
293 break;
294 }
295 }
296 free(fds);
297 return err;
298 }
299
300 static const struct got_error *
301 open_tempfiles(int **fds, size_t array_size, size_t nfds)
302 {
303 const struct got_error *err = NULL;
304 int i;
305
306 *fds = calloc(array_size, sizeof(**fds));
307 if (*fds == NULL)
308 return got_error_from_errno("calloc");
309
310 for (i = 0; i < array_size; i++)
311 (*fds)[i] = -1;
312
313 for (i = 0; i < nfds; i++) {
314 (*fds)[i] = got_opentempfd();
315 if ((*fds)[i] == -1) {
316 err = got_error_from_errno("got_opentempfd");
317 close_tempfiles(*fds, nfds);
318 *fds = NULL;
319 return err;
320 }
321 }
322
323 return NULL;
324 }
325
326 static const struct got_error *
327 get_pack_cache_size(int *pack_cache_size)
328 {
329 struct rlimit rl;
330
331 if (getrlimit(RLIMIT_NOFILE, &rl) == -1)
332 return got_error_from_errno("getrlimit");
333
334 *pack_cache_size = GOT_PACK_CACHE_SIZE;
335 if (*pack_cache_size > rl.rlim_cur / 8)
336 *pack_cache_size = rl.rlim_cur / 8;
337
338 return NULL;
339 }
340
341 const struct got_error *
342 got_repo_pack_fds_open(int **pack_fds)
343 {
344 const struct got_error *err;
345 int nfds;
346
347 err = get_pack_cache_size(&nfds);
348 if (err)
349 return err;
350
351 /*
352 * We need one basefd and one accumfd per cached pack.
353 * Our constants should be set up in a way such that
354 * this error never triggers.
355 */
356 if (nfds * 2 > GOT_PACK_NUM_TEMPFILES)
357 return got_error(GOT_ERR_NO_SPACE);
358
359 return open_tempfiles(pack_fds, GOT_PACK_NUM_TEMPFILES, nfds * 2);
360 }
361
362 const struct got_error *
363 got_repo_pack_fds_close(int *pack_fds)
364 {
365 return close_tempfiles(pack_fds, GOT_PACK_NUM_TEMPFILES);
366 }
367
368 const struct got_error *
369 got_repo_temp_fds_open(int **temp_fds)
370 {
371 return open_tempfiles(temp_fds, GOT_REPO_NUM_TEMPFILES,
372 GOT_REPO_NUM_TEMPFILES);
373 }
374
375 void
376 got_repo_temp_fds_set(struct got_repository *repo, int *temp_fds)
377 {
378 int i;
379
380 for (i = 0; i < GOT_REPO_NUM_TEMPFILES; i++)
381 repo->tempfiles[i] = temp_fds[i];
382 }
383
384 const struct got_error *
385 got_repo_temp_fds_get(int *fd, int *idx, struct got_repository *repo)
386 {
387 int i;
388
389 *fd = -1;
390 *idx = -1;
391
392 for (i = 0; i < nitems(repo->tempfiles); i++) {
393 if (repo->tempfile_use_mask & (1 << i))
394 continue;
395 if (repo->tempfiles[i] != -1) {
396 if (ftruncate(repo->tempfiles[i], 0L) == -1)
397 return got_error_from_errno("ftruncate");
398 *fd = repo->tempfiles[i];
399 *idx = i;
400 repo->tempfile_use_mask |= (1 << i);
401 return NULL;
402 }
403 }
404
405 return got_error(GOT_ERR_REPO_TEMPFILE);
406 }
407
408 void
409 got_repo_temp_fds_put(int idx, struct got_repository *repo)
410 {
411 repo->tempfile_use_mask &= ~(1 << idx);
412 }
413
414 const struct got_error *
415 got_repo_temp_fds_close(int *temp_fds)
416 {
417 return close_tempfiles(temp_fds, GOT_REPO_NUM_TEMPFILES);
418 }
419
420 const struct got_error *
421 got_repo_cache_object(struct got_repository *repo, struct got_object_id *id,
422 struct got_object *obj)
423 {
424 #ifndef GOT_NO_OBJ_CACHE
425 const struct got_error *err = NULL;
426 err = got_object_cache_add(&repo->objcache, id, obj);
427 if (err) {
428 if (err->code == GOT_ERR_OBJ_EXISTS ||
429 err->code == GOT_ERR_OBJ_TOO_LARGE)
430 err = NULL;
431 return err;
432 }
433 obj->refcnt++;
434 #endif
435 return NULL;
436 }
437
438 struct got_object *
439 got_repo_get_cached_object(struct got_repository *repo,
440 struct got_object_id *id)
441 {
442 return (struct got_object *)got_object_cache_get(&repo->objcache, id);
443 }
444
445 const struct got_error *
446 got_repo_cache_tree(struct got_repository *repo, struct got_object_id *id,
447 struct got_tree_object *tree)
448 {
449 #ifndef GOT_NO_OBJ_CACHE
450 const struct got_error *err = NULL;
451 err = got_object_cache_add(&repo->treecache, id, tree);
452 if (err) {
453 if (err->code == GOT_ERR_OBJ_EXISTS ||
454 err->code == GOT_ERR_OBJ_TOO_LARGE)
455 err = NULL;
456 return err;
457 }
458 tree->refcnt++;
459 #endif
460 return NULL;
461 }
462
463 struct got_tree_object *
464 got_repo_get_cached_tree(struct got_repository *repo,
465 struct got_object_id *id)
466 {
467 return (struct got_tree_object *)got_object_cache_get(
468 &repo->treecache, id);
469 }
470
471 const struct got_error *
472 got_repo_cache_commit(struct got_repository *repo, struct got_object_id *id,
473 struct got_commit_object *commit)
474 {
475 #ifndef GOT_NO_OBJ_CACHE
476 const struct got_error *err = NULL;
477 err = got_object_cache_add(&repo->commitcache, id, commit);
478 if (err) {
479 if (err->code == GOT_ERR_OBJ_EXISTS ||
480 err->code == GOT_ERR_OBJ_TOO_LARGE)
481 err = NULL;
482 return err;
483 }
484 commit->refcnt++;
485 #endif
486 return NULL;
487 }
488
489 struct got_commit_object *
490 got_repo_get_cached_commit(struct got_repository *repo,
491 struct got_object_id *id)
492 {
493 return (struct got_commit_object *)got_object_cache_get(
494 &repo->commitcache, id);
495 }
496
497 const struct got_error *
498 got_repo_cache_tag(struct got_repository *repo, struct got_object_id *id,
499 struct got_tag_object *tag)
500 {
501 #ifndef GOT_NO_OBJ_CACHE
502 const struct got_error *err = NULL;
503 err = got_object_cache_add(&repo->tagcache, id, tag);
504 if (err) {
505 if (err->code == GOT_ERR_OBJ_EXISTS ||
506 err->code == GOT_ERR_OBJ_TOO_LARGE)
507 err = NULL;
508 return err;
509 }
510 tag->refcnt++;
511 #endif
512 return NULL;
513 }
514
515 struct got_tag_object *
516 got_repo_get_cached_tag(struct got_repository *repo, struct got_object_id *id)
517 {
518 return (struct got_tag_object *)got_object_cache_get(
519 &repo->tagcache, id);
520 }
521
522 const struct got_error *
523 got_repo_cache_raw_object(struct got_repository *repo, struct got_object_id *id,
524 struct got_raw_object *raw)
525 {
526 #ifndef GOT_NO_OBJ_CACHE
527 const struct got_error *err = NULL;
528 err = got_object_cache_add(&repo->rawcache, id, raw);
529 if (err) {
530 if (err->code == GOT_ERR_OBJ_EXISTS ||
531 err->code == GOT_ERR_OBJ_TOO_LARGE)
532 err = NULL;
533 return err;
534 }
535 raw->refcnt++;
536 #endif
537 return NULL;
538 }
539
540
541 struct got_raw_object *
542 got_repo_get_cached_raw_object(struct got_repository *repo,
543 struct got_object_id *id)
544 {
545 return (struct got_raw_object *)got_object_cache_get(&repo->rawcache, id);
546 }
547
548
549 static const struct got_error *
550 open_repo(struct got_repository *repo, const char *path)
551 {
552 const struct got_error *err = NULL;
553
554 repo->gitdir_fd = -1;
555
556 /* bare git repository? */
557 repo->path_git_dir = strdup(path);
558 if (repo->path_git_dir == NULL)
559 return got_error_from_errno("strdup");
560 if (is_git_repo(repo)) {
561 repo->path = strdup(repo->path_git_dir);
562 if (repo->path == NULL) {
563 err = got_error_from_errno("strdup");
564 goto done;
565 }
566 repo->gitdir_fd = open(repo->path_git_dir,
567 O_DIRECTORY | O_CLOEXEC);
568 if (repo->gitdir_fd == -1) {
569 err = got_error_from_errno2("open",
570 repo->path_git_dir);
571 goto done;
572 }
573 return NULL;
574 }
575
576 /* git repository with working tree? */
577 free(repo->path_git_dir);
578 repo->path_git_dir = NULL;
579 if (asprintf(&repo->path_git_dir, "%s/%s", path, GOT_GIT_DIR) == -1) {
580 err = got_error_from_errno("asprintf");
581 goto done;
582 }
583 if (is_git_repo(repo)) {
584 repo->path = strdup(path);
585 if (repo->path == NULL) {
586 err = got_error_from_errno("strdup");
587 goto done;
588 }
589 repo->gitdir_fd = open(repo->path_git_dir,
590 O_DIRECTORY | O_CLOEXEC);
591 if (repo->gitdir_fd == -1) {
592 err = got_error_from_errno2("open",
593 repo->path_git_dir);
594 goto done;
595 }
596 return NULL;
597 }
598
599 err = got_error(GOT_ERR_NOT_GIT_REPO);
600 done:
601 if (err) {
602 free(repo->path);
603 repo->path = NULL;
604 free(repo->path_git_dir);
605 repo->path_git_dir = NULL;
606 if (repo->gitdir_fd != -1)
607 close(repo->gitdir_fd);
608 repo->gitdir_fd = -1;
609
610 }
611 return err;
612 }
613
614 static const struct got_error *
615 read_gitconfig(struct got_repository *repo, const char *global_gitconfig_path)
616 {
617 const struct got_error *err = NULL;
618 char *repo_gitconfig_path = NULL;
619
620 if (global_gitconfig_path) {
621 /* Read settings from ~/.gitconfig. */
622 int dummy_repo_version;
623 err = got_repo_read_gitconfig(&dummy_repo_version,
624 &repo->global_gitconfig_author_name,
625 &repo->global_gitconfig_author_email,
626 NULL, NULL, NULL, NULL, NULL, NULL,
627 global_gitconfig_path);
628 if (err)
629 return err;
630 }
631
632 /* Read repository's .git/config file. */
633 repo_gitconfig_path = got_repo_get_path_gitconfig(repo);
634 if (repo_gitconfig_path == NULL)
635 return got_error_from_errno("got_repo_get_path_gitconfig");
636
637 err = got_repo_read_gitconfig(
638 &repo->gitconfig_repository_format_version,
639 &repo->gitconfig_author_name, &repo->gitconfig_author_email,
640 &repo->gitconfig_remotes, &repo->ngitconfig_remotes,
641 &repo->gitconfig_owner, &repo->extnames, &repo->extvals,
642 &repo->nextensions, repo_gitconfig_path);
643 if (err)
644 goto done;
645
646 if (getenv("GOT_IGNORE_GITCONFIG") != NULL) {
647 int i;
648
649 for (i = 0; i < repo->ngitconfig_remotes; i++) {
650 got_repo_free_remote_repo_data(
651 &repo->gitconfig_remotes[i]);
652 }
653 free(repo->gitconfig_remotes);
654 repo->gitconfig_remotes = NULL;
655 repo->ngitconfig_remotes = 0;
656
657 free(repo->gitconfig_author_name);
658 repo->gitconfig_author_name = NULL;
659 free(repo->gitconfig_author_email);
660 repo->gitconfig_author_email = NULL;
661
662 free(repo->global_gitconfig_author_name);
663 repo->global_gitconfig_author_name = NULL;
664 free(repo->global_gitconfig_author_email);
665 repo->global_gitconfig_author_email = NULL;
666 }
667
668 done:
669 free(repo_gitconfig_path);
670 return err;
671 }
672
673 static const struct got_error *
674 read_gotconfig(struct got_repository *repo)
675 {
676 const struct got_error *err = NULL;
677 char *gotconfig_path;
678
679 gotconfig_path = got_repo_get_path_gotconfig(repo);
680 if (gotconfig_path == NULL)
681 return got_error_from_errno("got_repo_get_path_gotconfig");
682
683 err = got_gotconfig_read(&repo->gotconfig, gotconfig_path);
684 free(gotconfig_path);
685 return err;
686 }
687
688 /* Supported repository format extensions. */
689 static const char *const repo_extensions[] = {
690 "noop", /* Got supports repository format version 1. */
691 "preciousObjects", /* Supported by gotadmin cleanup. */
692 "worktreeConfig", /* Got does not care about Git work trees. */
693 };
694
695 const struct got_error *
696 got_repo_open(struct got_repository **repop, const char *path,
697 const char *global_gitconfig_path, int *pack_fds)
698 {
699 struct got_repository *repo = NULL;
700 const struct got_error *err = NULL;
701 char *repo_path = NULL;
702 size_t i, j = 0;
703
704 *repop = NULL;
705
706 repo = calloc(1, sizeof(*repo));
707 if (repo == NULL)
708 return got_error_from_errno("calloc");
709
710 RB_INIT(&repo->packidx_bloom_filters);
711 RB_INIT(&repo->packidx_paths);
712
713 for (i = 0; i < nitems(repo->privsep_children); i++) {
714 memset(&repo->privsep_children[i], 0,
715 sizeof(repo->privsep_children[0]));
716 repo->privsep_children[i].imsg_fd = -1;
717 }
718
719 err = got_object_cache_init(&repo->objcache,
720 GOT_OBJECT_CACHE_TYPE_OBJ);
721 if (err)
722 goto done;
723 err = got_object_cache_init(&repo->treecache,
724 GOT_OBJECT_CACHE_TYPE_TREE);
725 if (err)
726 goto done;
727 err = got_object_cache_init(&repo->commitcache,
728 GOT_OBJECT_CACHE_TYPE_COMMIT);
729 if (err)
730 goto done;
731 err = got_object_cache_init(&repo->tagcache,
732 GOT_OBJECT_CACHE_TYPE_TAG);
733 if (err)
734 goto done;
735 err = got_object_cache_init(&repo->rawcache,
736 GOT_OBJECT_CACHE_TYPE_RAW);
737 if (err)
738 goto done;
739
740 err = get_pack_cache_size(&repo->pack_cache_size);
741 if (err)
742 goto done;
743 for (i = 0; i < nitems(repo->packs); i++) {
744 if (pack_fds != NULL && i < repo->pack_cache_size) {
745 repo->packs[i].basefd = pack_fds[j++];
746 repo->packs[i].accumfd = pack_fds[j++];
747 } else {
748 repo->packs[i].basefd = -1;
749 repo->packs[i].accumfd = -1;
750 }
751 }
752 for (i = 0; i < nitems(repo->tempfiles); i++)
753 repo->tempfiles[i] = -1;
754 repo->pinned_pack = -1;
755 repo->pinned_packidx = -1;
756 repo->pinned_pid = 0;
757
758 repo_path = realpath(path, NULL);
759 if (repo_path == NULL) {
760 err = got_error_from_errno2("realpath", path);
761 goto done;
762 }
763
764 for (;;) {
765 char *parent_path;
766
767 err = open_repo(repo, repo_path);
768 if (err == NULL)
769 break;
770 if (err->code != GOT_ERR_NOT_GIT_REPO)
771 goto done;
772 if (repo_path[0] == '/' && repo_path[1] == '\0') {
773 err = got_error(GOT_ERR_NOT_GIT_REPO);
774 goto done;
775 }
776 err = got_path_dirname(&parent_path, repo_path);
777 if (err)
778 goto done;
779 free(repo_path);
780 repo_path = parent_path;
781 }
782
783 err = read_gotconfig(repo);
784 if (err)
785 goto done;
786
787 err = read_gitconfig(repo, global_gitconfig_path);
788 if (err)
789 goto done;
790 if (repo->gitconfig_repository_format_version > 1) {
791 err = got_error_path(path, GOT_ERR_GIT_REPO_FORMAT);
792 goto done;
793 }
794 for (i = 0; i < repo->nextensions; i++) {
795 char *ext = repo->extnames[i];
796 char *val = repo->extvals[i];
797 int j, supported = 0;
798
799 if (repo->gitconfig_repository_format_version == 1 &&
800 strcasecmp(ext, "objectformat") == 0) {
801 if (strcmp(val, "sha1") == 0)
802 continue;
803 if (strcmp(val, "sha256") == 0) {
804 repo->algo = GOT_HASH_SHA256;
805 continue;
806 }
807 err = got_error_path(val, GOT_ERR_OBJECT_FORMAT);
808 goto done;
809 }
810
811 if (!is_boolean_val(val)) {
812 err = got_error_path(ext, GOT_ERR_GIT_REPO_EXT);
813 goto done;
814 }
815
816 if (!get_boolean_val(val))
817 continue;
818
819 for (j = 0; j < nitems(repo_extensions); j++) {
820 if (strcmp(ext, repo_extensions[j]) == 0) {
821 supported = 1;
822 break;
823 }
824 }
825 if (!supported) {
826 err = got_error_path(ext, GOT_ERR_GIT_REPO_EXT);
827 goto done;
828 }
829 }
830
831 err = got_repo_list_packidx(&repo->packidx_paths, repo);
832 done:
833 if (err)
834 got_repo_close(repo);
835 else
836 *repop = repo;
837 free(repo_path);
838 return err;
839 }
840
841 const struct got_error *
842 got_repo_close(struct got_repository *repo)
843 {
844 const struct got_error *err = NULL, *child_err;
845 struct got_packidx_bloom_filter *bf;
846 size_t i;
847
848 for (i = 0; i < repo->pack_cache_size; i++) {
849 if (repo->packidx_cache[i] == NULL)
850 break;
851 got_packidx_close(repo->packidx_cache[i]);
852 }
853
854 while ((bf = RB_MIN(got_packidx_bloom_filter_tree,
855 &repo->packidx_bloom_filters))) {
856 RB_REMOVE(got_packidx_bloom_filter_tree,
857 &repo->packidx_bloom_filters, bf);
858 bloom_free(bf->bloom);
859 free(bf->bloom);
860 free(bf);
861 }
862
863 for (i = 0; i < repo->pack_cache_size; i++)
864 if (repo->packs[i].path_packfile)
865 if (repo->packs[i].path_packfile)
866 got_pack_close(&repo->packs[i]);
867
868 free(repo->path);
869 free(repo->path_git_dir);
870
871 got_object_cache_close(&repo->objcache);
872 got_object_cache_close(&repo->treecache);
873 got_object_cache_close(&repo->commitcache);
874 got_object_cache_close(&repo->tagcache);
875 got_object_cache_close(&repo->rawcache);
876
877 for (i = 0; i < nitems(repo->privsep_children); i++) {
878 if (repo->privsep_children[i].imsg_fd == -1)
879 continue;
880 imsgbuf_clear(repo->privsep_children[i].ibuf);
881 free(repo->privsep_children[i].ibuf);
882 err = got_privsep_send_stop(repo->privsep_children[i].imsg_fd);
883 if (err && err->code == GOT_ERR_EOF)
884 err = NULL;
885 child_err = got_privsep_wait_for_child(
886 repo->privsep_children[i].pid);
887 if (child_err && err == NULL)
888 err = child_err;
889 if (close(repo->privsep_children[i].imsg_fd) == -1 &&
890 err == NULL)
891 err = got_error_from_errno("close");
892 }
893
894 if (repo->gitdir_fd != -1 && close(repo->gitdir_fd) == -1 &&
895 err == NULL)
896 err = got_error_from_errno("close");
897
898 if (repo->gotconfig)
899 got_gotconfig_free(repo->gotconfig);
900 free(repo->gitconfig_author_name);
901 free(repo->gitconfig_author_email);
902 for (i = 0; i < repo->ngitconfig_remotes; i++)
903 got_repo_free_remote_repo_data(&repo->gitconfig_remotes[i]);
904 free(repo->gitconfig_remotes);
905 for (i = 0; i < repo->nextensions; i++) {
906 free(repo->extnames[i]);
907 free(repo->extvals[i]);
908 }
909 free(repo->extnames);
910 free(repo->extvals);
911
912 got_pathlist_free(&repo->packidx_paths, GOT_PATHLIST_FREE_PATH);
913 free(repo);
914
915 return err;
916 }
917
918 const struct got_error *
919 got_repo_remote_repo_dup(struct got_remote_repo **newp,
920 const struct got_remote_repo *repo)
921 {
922 const struct got_error *err = NULL;
923 struct got_remote_repo *new;
924 int i;
925
926 new = calloc(1, sizeof(*new));
927 if (new == NULL)
928 return got_error_from_errno("calloc");
929
930 if (repo->name) {
931 new->name = strdup(repo->name);
932 if (new->name == NULL) {
933 err = got_error_from_errno("strdup");
934 goto done;
935 }
936 }
937
938 if (repo->fetch_url) {
939 new->fetch_url = strdup(repo->fetch_url);
940 if (new->fetch_url == NULL) {
941 err = got_error_from_errno("strdup");
942 goto done;
943 }
944 }
945
946 if (repo->send_url) {
947 new->send_url = strdup(repo->send_url);
948 if (new->send_url == NULL) {
949 err = got_error_from_errno("strdup");
950 goto done;
951 }
952 }
953
954 new->mirror_references = repo->mirror_references;
955
956 new->fetch_all_branches = repo->fetch_all_branches;
957
958 new->nfetch_branches = repo->nfetch_branches;
959 if (repo->fetch_branches) {
960 new->fetch_branches = calloc(repo->nfetch_branches,
961 sizeof(char *));
962 if (new->fetch_branches == NULL) {
963 err = got_error_from_errno("calloc");
964 goto done;
965 }
966 for (i = 0; i < repo->nfetch_branches; i++) {
967 new->fetch_branches[i] = strdup(
968 repo->fetch_branches[i]);
969 if (new->fetch_branches[i] == NULL) {
970 err = got_error_from_errno("strdup");
971 goto done;
972 }
973 }
974 }
975
976 new->nsend_branches = repo->nsend_branches;
977 if (repo->send_branches) {
978 new->send_branches = calloc(repo->nsend_branches,
979 sizeof(char *));
980 if (new->send_branches == NULL) {
981 err = got_error_from_errno("calloc");
982 goto done;
983 }
984 for (i = 0; i < repo->nsend_branches; i++) {
985 new->send_branches[i] = strdup(
986 repo->send_branches[i]);
987 if (new->send_branches[i] == NULL) {
988 err = got_error_from_errno("strdup");
989 goto done;
990 }
991 }
992 }
993
994 new->nfetch_refs = repo->nfetch_refs;
995 if (repo->fetch_refs) {
996 new->fetch_refs = calloc(repo->nfetch_refs,
997 sizeof(char *));
998 if (new->fetch_refs == NULL) {
999 err = got_error_from_errno("calloc");
1000 goto done;
1001 }
1002 for (i = 0; i < repo->nfetch_refs; i++) {
1003 new->fetch_refs[i] = strdup(
1004 repo->fetch_refs[i]);
1005 if (new->fetch_refs[i] == NULL) {
1006 err = got_error_from_errno("strdup");
1007 goto done;
1008 }
1009 }
1010 }
1011 done:
1012 if (err) {
1013 got_repo_free_remote_repo_data(new);
1014 free(new);
1015 } else
1016 *newp = new;
1017
1018 return err;
1019 }
1020
1021 void
1022 got_repo_free_remote_repo_data(struct got_remote_repo *repo)
1023 {
1024 int i;
1025
1026 if (repo == NULL)
1027 return;
1028
1029 free(repo->name);
1030 repo->name = NULL;
1031 free(repo->fetch_url);
1032 repo->fetch_url = NULL;
1033 free(repo->send_url);
1034 repo->send_url = NULL;
1035 for (i = 0; i < repo->nfetch_branches; i++)
1036 free(repo->fetch_branches[i]);
1037 free(repo->fetch_branches);
1038 repo->fetch_branches = NULL;
1039 repo->nfetch_branches = 0;
1040 for (i = 0; i < repo->nsend_branches; i++)
1041 free(repo->send_branches[i]);
1042 free(repo->send_branches);
1043 repo->send_branches = NULL;
1044 repo->nsend_branches = 0;
1045 for (i = 0; i < repo->nfetch_refs; i++)
1046 free(repo->fetch_refs[i]);
1047 free(repo->fetch_refs);
1048 repo->fetch_refs = NULL;
1049 repo->nfetch_refs = 0;
1050 }
1051
1052 const struct got_error *
1053 got_repo_map_path(char **in_repo_path, struct got_repository *repo,
1054 const char *input_path)
1055 {
1056 const struct got_error *err = NULL;
1057 const char *repo_abspath = NULL;
1058 size_t repolen, len;
1059 char *canonpath, *path = NULL;
1060
1061 *in_repo_path = NULL;
1062
1063 canonpath = strdup(input_path);
1064 if (canonpath == NULL) {
1065 err = got_error_from_errno("strdup");
1066 goto done;
1067 }
1068 err = got_canonpath(input_path, canonpath, strlen(canonpath) + 1);
1069 if (err)
1070 goto done;
1071
1072 repo_abspath = got_repo_get_path(repo);
1073
1074 if (canonpath[0] == '\0') {
1075 path = strdup(canonpath);
1076 if (path == NULL) {
1077 err = got_error_from_errno("strdup");
1078 goto done;
1079 }
1080 } else {
1081 path = realpath(canonpath, NULL);
1082 if (path == NULL) {
1083 if (errno != ENOENT) {
1084 err = got_error_from_errno2("realpath",
1085 canonpath);
1086 goto done;
1087 }
1088 /*
1089 * Path is not on disk.
1090 * Assume it is already relative to repository root.
1091 */
1092 path = strdup(canonpath);
1093 if (path == NULL) {
1094 err = got_error_from_errno("strdup");
1095 goto done;
1096 }
1097 }
1098
1099 repolen = strlen(repo_abspath);
1100 len = strlen(path);
1101
1102
1103 if (strcmp(path, repo_abspath) == 0) {
1104 free(path);
1105 path = strdup("");
1106 if (path == NULL) {
1107 err = got_error_from_errno("strdup");
1108 goto done;
1109 }
1110 } else if (len > repolen &&
1111 got_path_is_child(path, repo_abspath, repolen)) {
1112 /* Matched an on-disk path inside repository. */
1113 if (got_repo_is_bare(repo)) {
1114 /*
1115 * Matched an on-disk path inside repository
1116 * database. Treat input as repository-relative.
1117 */
1118 free(path);
1119 path = canonpath;
1120 canonpath = NULL;
1121 } else {
1122 char *child;
1123 /* Strip common prefix with repository path. */
1124 err = got_path_skip_common_ancestor(&child,
1125 repo_abspath, path);
1126 if (err)
1127 goto done;
1128 free(path);
1129 path = child;
1130 }
1131 } else {
1132 /*
1133 * Matched unrelated on-disk path.
1134 * Treat input as repository-relative.
1135 */
1136 free(path);
1137 path = canonpath;
1138 canonpath = NULL;
1139 }
1140 }
1141
1142 /* Make in-repository path absolute */
1143 if (path[0] != '/') {
1144 char *abspath;
1145 if (asprintf(&abspath, "/%s", path) == -1) {
1146 err = got_error_from_errno("asprintf");
1147 goto done;
1148 }
1149 free(path);
1150 path = abspath;
1151 }
1152
1153 done:
1154 free(canonpath);
1155 if (err)
1156 free(path);
1157 else
1158 *in_repo_path = path;
1159 return err;
1160 }
1161
1162 static const struct got_error *
1163 cache_packidx(struct got_repository *repo, struct got_packidx *packidx,
1164 const char *path_packidx)
1165 {
1166 const struct got_error *err = NULL;
1167 size_t i;
1168
1169 for (i = 0; i < repo->pack_cache_size; i++) {
1170 if (repo->packidx_cache[i] == NULL)
1171 break;
1172 if (strcmp(repo->packidx_cache[i]->path_packidx,
1173 path_packidx) == 0) {
1174 return got_error(GOT_ERR_CACHE_DUP_ENTRY);
1175 }
1176 }
1177 if (i == repo->pack_cache_size) {
1178 do {
1179 i--;
1180 } while (i > 0 && repo->pinned_packidx >= 0 &&
1181 i == repo->pinned_packidx);
1182 err = got_packidx_close(repo->packidx_cache[i]);
1183 if (err)
1184 return err;
1185 }
1186
1187 repo->packidx_cache[i] = packidx;
1188
1189 return NULL;
1190 }
1191
1192 int
1193 got_repo_is_packidx_filename(const char *name, size_t len,
1194 enum got_hash_algorithm algo)
1195 {
1196 size_t digest_string_len;
1197
1198 digest_string_len = got_hash_digest_string_length(algo);
1199
1200 if (len != GOT_PACKIDX_NAMELEN(digest_string_len))
1201 return 0;
1202
1203 if (strncmp(name, GOT_PACK_PREFIX, strlen(GOT_PACK_PREFIX)) != 0)
1204 return 0;
1205
1206 if (strcmp(name + strlen(GOT_PACK_PREFIX) + digest_string_len - 1,
1207 GOT_PACKIDX_SUFFIX) != 0)
1208 return 0;
1209
1210 return 1;
1211 }
1212
1213 static struct got_packidx_bloom_filter *
1214 get_packidx_bloom_filter(struct got_repository *repo,
1215 const char *path, size_t path_len)
1216 {
1217 struct got_packidx_bloom_filter key;
1218
1219 if (strlcpy(key.path, path, sizeof(key.path)) >= sizeof(key.path))
1220 return NULL; /* XXX */
1221 key.path_len = path_len;
1222
1223 return RB_FIND(got_packidx_bloom_filter_tree,
1224 &repo->packidx_bloom_filters, &key);
1225 }
1226
1227 int
1228 got_repo_check_packidx_bloom_filter(struct got_repository *repo,
1229 const char *path_packidx, struct got_object_id *id)
1230 {
1231 struct got_packidx_bloom_filter *bf;
1232
1233 bf = get_packidx_bloom_filter(repo, path_packidx, strlen(path_packidx));
1234 if (bf)
1235 return bloom_check(bf->bloom, id->hash,
1236 got_hash_digest_length(id->algo));
1237
1238 /* No bloom filter means this pack index must be searched. */
1239 return 1;
1240 }
1241
1242 static const struct got_error *
1243 add_packidx_bloom_filter(struct got_repository *repo,
1244 struct got_packidx *packidx, const char *path_packidx)
1245 {
1246 int i, nobjects = be32toh(packidx->hdr.fanout_table[0xff]);
1247 struct got_packidx_bloom_filter *bf;
1248 size_t len, digest_len;
1249
1250 digest_len = got_hash_digest_length(repo->algo);
1251
1252 /*
1253 * Don't use bloom filters for very large pack index files.
1254 * Large pack files will contain a relatively large fraction
1255 * of our objects so we will likely need to visit them anyway.
1256 * The more objects a pack file contains the higher the probability
1257 * of a false-positive match from the bloom filter. And reading
1258 * all object IDs from a large pack index file can be expensive.
1259 */
1260 if (nobjects > 100000) /* cut-off at about 2MB, at 20 bytes per ID */
1261 return NULL;
1262
1263 /* Do we already have a filter for this pack index? */
1264 if (get_packidx_bloom_filter(repo, path_packidx,
1265 strlen(path_packidx)) != NULL)
1266 return NULL;
1267
1268 bf = calloc(1, sizeof(*bf));
1269 if (bf == NULL)
1270 return got_error_from_errno("calloc");
1271 bf->bloom = calloc(1, sizeof(*bf->bloom));
1272 if (bf->bloom == NULL) {
1273 free(bf);
1274 return got_error_from_errno("calloc");
1275 }
1276
1277 len = strlcpy(bf->path, path_packidx, sizeof(bf->path));
1278 if (len >= sizeof(bf->path)) {
1279 free(bf->bloom);
1280 free(bf);
1281 return got_error(GOT_ERR_NO_SPACE);
1282 }
1283 bf->path_len = len;
1284
1285 /* Minimum size supported by our bloom filter is 1000 entries. */
1286 bloom_init(bf->bloom, nobjects < 1000 ? 1000 : nobjects, 0.1);
1287 for (i = 0; i < nobjects; i++) {
1288 uint8_t *id = packidx->hdr.sorted_ids + i * digest_len;
1289 bloom_add(bf->bloom, id, digest_len);
1290 }
1291
1292 RB_INSERT(got_packidx_bloom_filter_tree,
1293 &repo->packidx_bloom_filters, bf);
1294 return NULL;
1295 }
1296
1297 static void
1298 purge_packidx_paths(struct got_pathlist_head *packidx_paths)
1299 {
1300 struct got_pathlist_entry *pe;
1301
1302 while (!RB_EMPTY(packidx_paths)) {
1303 pe = RB_MIN(got_pathlist_head, packidx_paths);
1304 RB_REMOVE(got_pathlist_head, packidx_paths, pe);
1305 free((char *)pe->path);
1306 free(pe);
1307 }
1308 }
1309
1310 static const struct got_error *
1311 refresh_packidx_paths(struct got_repository *repo)
1312 {
1313 const struct got_error *err = NULL;
1314 char *objects_pack_dir = NULL;
1315 struct stat sb;
1316
1317 objects_pack_dir = got_repo_get_path_objects_pack(repo);
1318 if (objects_pack_dir == NULL)
1319 return got_error_from_errno("got_repo_get_path_objects_pack");
1320
1321 if (stat(objects_pack_dir, &sb) == -1) {
1322 if (errno != ENOENT) {
1323 err = got_error_from_errno2("stat", objects_pack_dir);
1324 goto done;
1325 }
1326 } else if (RB_EMPTY(&repo->packidx_paths) ||
1327 sb.st_mtim.tv_sec != repo->pack_path_mtime.tv_sec ||
1328 sb.st_mtim.tv_nsec != repo->pack_path_mtime.tv_nsec) {
1329 purge_packidx_paths(&repo->packidx_paths);
1330 err = got_repo_list_packidx(&repo->packidx_paths, repo);
1331 if (err)
1332 goto done;
1333 }
1334 done:
1335 free(objects_pack_dir);
1336 return err;
1337 }
1338
1339 const struct got_error *
1340 got_repo_search_packidx(struct got_packidx **packidx, int *idx,
1341 struct got_repository *repo, struct got_object_id *id)
1342 {
1343 const struct got_error *err;
1344 struct got_pathlist_entry *pe;
1345 size_t i;
1346
1347 /* Search pack index cache. */
1348 for (i = 0; i < repo->pack_cache_size; i++) {
1349 if (repo->packidx_cache[i] == NULL)
1350 break;
1351 if (!got_repo_check_packidx_bloom_filter(repo,
1352 repo->packidx_cache[i]->path_packidx, id))
1353 continue; /* object will not be found in this index */
1354 *idx = got_packidx_get_object_idx(repo->packidx_cache[i], id);
1355 if (*idx != -1) {
1356 *packidx = repo->packidx_cache[i];
1357 /*
1358 * Move this cache entry to the front. Repeatedly
1359 * searching a wrong pack index can be expensive.
1360 */
1361 if (i > 0) {
1362 memmove(&repo->packidx_cache[1],
1363 &repo->packidx_cache[0],
1364 i * sizeof(repo->packidx_cache[0]));
1365 repo->packidx_cache[0] = *packidx;
1366 if (repo->pinned_packidx >= 0 &&
1367 repo->pinned_packidx < i)
1368 repo->pinned_packidx++;
1369 else if (repo->pinned_packidx == i)
1370 repo->pinned_packidx = 0;
1371 }
1372 return NULL;
1373 }
1374 }
1375 /* No luck. Search the filesystem. */
1376
1377 err = refresh_packidx_paths(repo);
1378 if (err)
1379 return err;
1380
1381 RB_FOREACH(pe, got_pathlist_head, &repo->packidx_paths) {
1382 const char *path_packidx = pe->path;
1383 int is_cached = 0;
1384
1385 if (!got_repo_check_packidx_bloom_filter(repo,
1386 pe->path, id))
1387 continue; /* object will not be found in this index */
1388
1389 for (i = 0; i < repo->pack_cache_size; i++) {
1390 if (repo->packidx_cache[i] == NULL)
1391 break;
1392 if (strcmp(repo->packidx_cache[i]->path_packidx,
1393 path_packidx) == 0) {
1394 is_cached = 1;
1395 break;
1396 }
1397 }
1398 if (is_cached)
1399 continue; /* already searched */
1400
1401 err = got_packidx_open(packidx, got_repo_get_fd(repo),
1402 path_packidx, 0, repo->algo);
1403 if (err) {
1404 if (err->code == GOT_ERR_LONELY_PACKIDX) {
1405 err = NULL;
1406 continue;
1407 }
1408 goto done;
1409 }
1410
1411 err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
1412 if (err)
1413 goto done;
1414
1415 err = cache_packidx(repo, *packidx, path_packidx);
1416 if (err)
1417 goto done;
1418
1419 *idx = got_packidx_get_object_idx(*packidx, id);
1420 if (*idx != -1) {
1421 err = NULL; /* found the object */
1422 goto done;
1423 }
1424 }
1425
1426 err = got_error_no_obj(id);
1427 done:
1428 return err;
1429 }
1430
1431 const struct got_error *
1432 got_repo_list_packidx(struct got_pathlist_head *packidx_paths,
1433 struct got_repository *repo)
1434 {
1435 const struct got_error *err = NULL;
1436 DIR *packdir = NULL;
1437 struct dirent *dent;
1438 char *path_packidx = NULL;
1439 int packdir_fd;
1440 struct stat sb;
1441
1442 packdir_fd = openat(got_repo_get_fd(repo),
1443 GOT_OBJECTS_PACK_DIR, O_DIRECTORY | O_CLOEXEC);
1444 if (packdir_fd == -1) {
1445 return got_error_from_errno_fmt("openat: %s/%s",
1446 got_repo_get_path_git_dir(repo),
1447 GOT_OBJECTS_PACK_DIR);
1448 }
1449
1450 packdir = fdopendir(packdir_fd);
1451 if (packdir == NULL) {
1452 err = got_error_from_errno("fdopendir");
1453 close(packdir_fd);
1454 goto done;
1455 }
1456
1457 if (fstat(packdir_fd, &sb) == -1) {
1458 err = got_error_from_errno("fstat");
1459 goto done;
1460 }
1461 repo->pack_path_mtime.tv_sec = sb.st_mtim.tv_sec;
1462 repo->pack_path_mtime.tv_nsec = sb.st_mtim.tv_nsec;
1463
1464 while ((dent = readdir(packdir)) != NULL) {
1465 if (!got_repo_is_packidx_filename(dent->d_name,
1466 strlen(dent->d_name),
1467 repo->algo))
1468 continue;
1469
1470 if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
1471 dent->d_name) == -1) {
1472 err = got_error_from_errno("asprintf");
1473 path_packidx = NULL;
1474 break;
1475 }
1476
1477 err = got_pathlist_insert(NULL, packidx_paths, path_packidx, NULL);
1478 if (err)
1479 break;
1480 }
1481 done:
1482 if (err)
1483 free(path_packidx);
1484 if (packdir && closedir(packdir) != 0 && err == NULL)
1485 err = got_error_from_errno("closedir");
1486 return err;
1487 }
1488
1489 const struct got_error *
1490 got_repo_get_packidx(struct got_packidx **packidx, const char *path_packidx,
1491 struct got_repository *repo)
1492 {
1493 const struct got_error *err;
1494 size_t i;
1495
1496 *packidx = NULL;
1497
1498 /* Search pack index cache. */
1499 for (i = 0; i < repo->pack_cache_size; i++) {
1500 if (repo->packidx_cache[i] == NULL)
1501 break;
1502 if (strcmp(repo->packidx_cache[i]->path_packidx,
1503 path_packidx) == 0) {
1504 *packidx = repo->packidx_cache[i];
1505 return NULL;
1506 }
1507 }
1508 /* No luck. Search the filesystem. */
1509
1510 err = got_packidx_open(packidx, got_repo_get_fd(repo),
1511 path_packidx, 0, repo->algo);
1512 if (err)
1513 return err;
1514
1515 err = add_packidx_bloom_filter(repo, *packidx, path_packidx);
1516 if (err)
1517 goto done;
1518
1519 err = cache_packidx(repo, *packidx, path_packidx);
1520 done:
1521 if (err) {
1522 got_packidx_close(*packidx);
1523 *packidx = NULL;
1524 }
1525 return err;
1526 }
1527
1528 static const struct got_error *
1529 read_packfile_hdr(int fd, struct got_packidx *packidx)
1530 {
1531 const struct got_error *err = NULL;
1532 uint32_t totobj = be32toh(packidx->hdr.fanout_table[0xff]);
1533 struct got_packfile_hdr hdr;
1534 ssize_t n;
1535
1536 n = read(fd, &hdr, sizeof(hdr));
1537 if (n < 0)
1538 return got_error_from_errno("read");
1539 if (n != sizeof(hdr))
1540 return got_error(GOT_ERR_BAD_PACKFILE);
1541
1542 if (be32toh(hdr.signature) != GOT_PACKFILE_SIGNATURE ||
1543 be32toh(hdr.version) != GOT_PACKFILE_VERSION ||
1544 be32toh(hdr.nobjects) != totobj)
1545 err = got_error(GOT_ERR_BAD_PACKFILE);
1546
1547 return err;
1548 }
1549
1550 static const struct got_error *
1551 open_packfile(int *fd, struct got_repository *repo,
1552 const char *relpath, struct got_packidx *packidx)
1553 {
1554 const struct got_error *err = NULL;
1555
1556 *fd = openat(got_repo_get_fd(repo), relpath,
1557 O_RDONLY | O_NOFOLLOW | O_CLOEXEC);
1558 if (*fd == -1)
1559 return got_error_from_errno_fmt("openat: %s/%s",
1560 got_repo_get_path_git_dir(repo), relpath);
1561
1562 if (packidx) {
1563 err = read_packfile_hdr(*fd, packidx);
1564 if (err) {
1565 close(*fd);
1566 *fd = -1;
1567 }
1568 }
1569
1570 return err;
1571 }
1572
1573 const struct got_error *
1574 got_repo_cache_pack(struct got_pack **packp, struct got_repository *repo,
1575 const char *path_packfile, struct got_packidx *packidx)
1576 {
1577 const struct got_error *err = NULL;
1578 struct got_pack *pack = NULL;
1579 struct got_pack tmp;
1580 struct stat sb;
1581 size_t i;
1582
1583 if (packp)
1584 *packp = NULL;
1585
1586 for (i = 0; i < repo->pack_cache_size; i++) {
1587 pack = &repo->packs[i];
1588 if (pack->path_packfile == NULL)
1589 break;
1590 if (strcmp(pack->path_packfile, path_packfile) == 0)
1591 return got_error(GOT_ERR_CACHE_DUP_ENTRY);
1592 }
1593
1594 if (i == repo->pack_cache_size) {
1595 do {
1596 i--;
1597 } while (i > 0 && repo->pinned_pack >= 0 &&
1598 i == repo->pinned_pack);
1599 err = got_pack_close(&repo->packs[i]);
1600 if (err)
1601 return err;
1602 if (ftruncate(repo->packs[i].basefd, 0L) == -1)
1603 return got_error_from_errno("ftruncate");
1604 if (ftruncate(repo->packs[i].accumfd, 0L) == -1)
1605 return got_error_from_errno("ftruncate");
1606 }
1607
1608 if (i != 0) {
1609 memcpy(&tmp, &repo->packs[i], sizeof(tmp));
1610 memcpy(&repo->packs[i], &repo->packs[0],
1611 sizeof(repo->packs[i]));
1612 memcpy(&repo->packs[0], &tmp, sizeof(repo->packs[0]));
1613 if (repo->pinned_pack == 0)
1614 repo->pinned_pack = i;
1615 else if (repo->pinned_pack == i)
1616 repo->pinned_pack = 0;
1617 i = 0;
1618 }
1619
1620 pack = &repo->packs[i];
1621
1622 pack->path_packfile = strdup(path_packfile);
1623 if (pack->path_packfile == NULL) {
1624 err = got_error_from_errno("strdup");
1625 goto done;
1626 }
1627
1628 err = open_packfile(&pack->fd, repo, path_packfile, packidx);
1629 if (err)
1630 goto done;
1631
1632 if (fstat(pack->fd, &sb) != 0) {
1633 err = got_error_from_errno("fstat");
1634 goto done;
1635 }
1636 pack->filesize = sb.st_size;
1637 pack->algo = repo->algo;
1638
1639 pack->privsep_child = NULL;
1640
1641 err = got_delta_cache_alloc(&pack->delta_cache);
1642 if (err)
1643 goto done;
1644
1645 #ifndef GOT_PACK_NO_MMAP
1646 if (pack->filesize > 0 && pack->filesize <= SIZE_MAX) {
1647 pack->map = mmap(NULL, pack->filesize, PROT_READ, MAP_PRIVATE,
1648 pack->fd, 0);
1649 if (pack->map == MAP_FAILED) {
1650 if (errno != ENOMEM) {
1651 err = got_error_from_errno("mmap");
1652 goto done;
1653 }
1654 pack->map = NULL; /* fall back to read(2) */
1655 }
1656 }
1657 #endif
1658 done:
1659 if (err) {
1660 if (pack)
1661 got_pack_close(pack);
1662 } else if (packp)
1663 *packp = pack;
1664 return err;
1665 }
1666
1667 struct got_pack *
1668 got_repo_get_cached_pack(struct got_repository *repo, const char *path_packfile)
1669 {
1670 struct got_pack *pack = NULL;
1671 size_t i;
1672
1673 for (i = 0; i < repo->pack_cache_size; i++) {
1674 pack = &repo->packs[i];
1675 if (pack->path_packfile == NULL)
1676 break;
1677 if (strcmp(pack->path_packfile, path_packfile) == 0)
1678 return pack;
1679 }
1680
1681 return NULL;
1682 }
1683
1684 const struct got_error *
1685 got_repo_pin_pack(struct got_repository *repo, struct got_packidx *packidx,
1686 struct got_pack *pack)
1687 {
1688 size_t i;
1689 int pinned_pack = -1, pinned_packidx = -1;
1690
1691 for (i = 0; i < repo->pack_cache_size; i++) {
1692 if (repo->packidx_cache[i] &&
1693 strcmp(repo->packidx_cache[i]->path_packidx,
1694 packidx->path_packidx) == 0)
1695 pinned_packidx = i;
1696 if (repo->packs[i].path_packfile &&
1697 strcmp(repo->packs[i].path_packfile,
1698 pack->path_packfile) == 0)
1699 pinned_pack = i;
1700 }
1701
1702 if (pinned_packidx == -1 || pinned_pack == -1)
1703 return got_error(GOT_ERR_PIN_PACK);
1704
1705 repo->pinned_pack = pinned_pack;
1706 repo->pinned_packidx = pinned_packidx;
1707 if (repo->packs[pinned_pack].privsep_child)
1708 repo->pinned_pid = repo->packs[pinned_pack].privsep_child->pid;
1709 return NULL;
1710 }
1711
1712 struct got_pack *
1713 got_repo_get_pinned_pack(struct got_repository *repo)
1714 {
1715 if (repo->pinned_pack >= 0 &&
1716 repo->pinned_pack < repo->pack_cache_size)
1717 return &repo->packs[repo->pinned_pack];
1718
1719 return NULL;
1720 }
1721
1722 void
1723 got_repo_unpin_pack(struct got_repository *repo)
1724 {
1725 repo->pinned_packidx = -1;
1726 repo->pinned_pack = -1;
1727 repo->pinned_pid = 0;
1728 }
1729
1730 const struct got_error *
1731 got_repo_init(const char *repo_path, const char *head_name,
1732 enum got_hash_algorithm algo)
1733 {
1734 const struct got_error *err = NULL;
1735 const char *dirnames[] = {
1736 GOT_OBJECTS_DIR,
1737 GOT_OBJECTS_PACK_DIR,
1738 GOT_REFS_DIR,
1739 };
1740 const char *description_str = "Unnamed repository; "
1741 "edit this file 'description' to name the repository.";
1742 const char *headref = "ref: refs/heads/";
1743 const char *gitconfig_sha1 = "[core]\n"
1744 "\trepositoryformatversion = 0\n"
1745 "\tfilemode = true\n"
1746 "\tbare = true\n";
1747 const char *gitconfig_sha256 = "[core]\n"
1748 "\trepositoryformatversion = 1\n"
1749 "\tfilemode = true\n"
1750 "\tbare = true\n"
1751 "[extensions]\n"
1752 "\tobjectformat = sha256\n";
1753 const char *gitconfig = gitconfig_sha1;
1754 char *headref_str, *path;
1755 size_t i;
1756
1757 if (algo == GOT_HASH_SHA256)
1758 gitconfig = gitconfig_sha256;
1759
1760 if (!got_path_dir_is_empty(repo_path))
1761 return got_error(GOT_ERR_DIR_NOT_EMPTY);
1762
1763 for (i = 0; i < nitems(dirnames); i++) {
1764 if (asprintf(&path, "%s/%s", repo_path, dirnames[i]) == -1) {
1765 return got_error_from_errno("asprintf");
1766 }
1767 err = got_path_mkdir(path);
1768 free(path);
1769 if (err)
1770 return err;
1771 }
1772
1773 if (asprintf(&path, "%s/%s", repo_path, "description") == -1)
1774 return got_error_from_errno("asprintf");
1775 err = got_path_create_file(path, description_str);
1776 free(path);
1777 if (err)
1778 return err;
1779
1780 if (asprintf(&path, "%s/%s", repo_path, GOT_HEAD_FILE) == -1)
1781 return got_error_from_errno("asprintf");
1782 if (asprintf(&headref_str, "%s%s", headref,
1783 head_name ? head_name : "main") == -1) {
1784 free(path);
1785 return got_error_from_errno("asprintf");
1786 }
1787 err = got_path_create_file(path, headref_str);
1788 free(headref_str);
1789 free(path);
1790 if (err)
1791 return err;
1792
1793 if (asprintf(&path, "%s/%s", repo_path, "config") == -1)
1794 return got_error_from_errno("asprintf");
1795 err = got_path_create_file(path, gitconfig);
1796 free(path);
1797 if (err)
1798 return err;
1799
1800 return NULL;
1801 }
1802
1803 static const struct got_error *
1804 match_packed_object(struct got_object_id **unique_id,
1805 struct got_repository *repo, const char *id_str_prefix, int obj_type)
1806 {
1807 const struct got_error *err = NULL;
1808 struct got_object_id_queue matched_ids;
1809 struct got_pathlist_entry *pe;
1810 struct timespec tv;
1811 int retries = 0;
1812 const int max_retries = 10;
1813
1814 STAILQ_INIT(&matched_ids);
1815
1816 err = refresh_packidx_paths(repo);
1817 if (err)
1818 return err;
1819
1820 /*
1821 * Opening objects while iterating over the pack-index path
1822 * list is racy. If the set of pack files in the repository
1823 * changes during loop iteration, refresh_packidx_paths() will
1824 * be called again, via got_object_get_type(), invalidating
1825 * the packidx_paths list we are iterating over.
1826 * To work around this we keep track of the current modification
1827 * time and retry the entire loop if it changes.
1828 */
1829 retry:
1830 tv.tv_sec = repo->pack_path_mtime.tv_sec;
1831 tv.tv_nsec = repo->pack_path_mtime.tv_nsec;
1832
1833 RB_FOREACH(pe, got_pathlist_head, &repo->packidx_paths) {
1834 const char *path_packidx;
1835 struct got_packidx *packidx;
1836 struct got_object_qid *qid;
1837
1838 /*
1839 * If the modification time of the 'objects/pack' directory
1840 * has changed then 'pe' could now be an invalid pointer.
1841 */
1842 if (tv.tv_sec != repo->pack_path_mtime.tv_sec ||
1843 tv.tv_nsec != repo->pack_path_mtime.tv_nsec) {
1844 if (++retries > max_retries) {
1845 err = got_error_msg(GOT_ERR_TIMEOUT,
1846 "too many concurrent pack file "
1847 "modifications");
1848 goto done;
1849 }
1850 goto retry;
1851 }
1852
1853 path_packidx = pe->path;
1854
1855 err = got_packidx_open(&packidx, got_repo_get_fd(repo),
1856 path_packidx, 0, repo->algo);
1857 if (err) {
1858 if (err->code == GOT_ERR_LONELY_PACKIDX) {
1859 err = NULL;
1860 continue;
1861 }
1862 break;
1863 }
1864
1865 got_object_id_queue_free(&matched_ids);
1866
1867 err = got_packidx_match_id_str_prefix(&matched_ids,
1868 packidx, id_str_prefix);
1869 if (err) {
1870 got_packidx_close(packidx);
1871 break;
1872 }
1873 err = got_packidx_close(packidx);
1874 if (err)
1875 break;
1876
1877 STAILQ_FOREACH(qid, &matched_ids, entry) {
1878 if (obj_type != GOT_OBJ_TYPE_ANY) {
1879 int matched_type;
1880 err = got_object_get_type(&matched_type, repo,
1881 &qid->id);
1882 if (err)
1883 goto done;
1884 if (matched_type != obj_type)
1885 continue;
1886 }
1887 if (*unique_id == NULL) {
1888 *unique_id = got_object_id_dup(&qid->id);
1889 if (*unique_id == NULL) {
1890 err = got_error_from_errno("malloc");
1891 goto done;
1892 }
1893 } else {
1894 if (got_object_id_cmp(*unique_id,
1895 &qid->id) == 0)
1896 continue; /* packed multiple times */
1897 err = got_error(GOT_ERR_AMBIGUOUS_ID);
1898 goto done;
1899 }
1900 }
1901 }
1902 done:
1903 got_object_id_queue_free(&matched_ids);
1904 if (err) {
1905 free(*unique_id);
1906 *unique_id = NULL;
1907 }
1908 return err;
1909 }
1910
1911 static const struct got_error *
1912 match_loose_object(struct got_object_id **unique_id, const char *path_objects,
1913 const char *object_dir, const char *id_str_prefix, int obj_type,
1914 struct got_repository *repo)
1915 {
1916 const struct got_error *err = NULL;
1917 char *path, *id_str = NULL;
1918 DIR *dir = NULL;
1919 struct dirent *dent;
1920 struct got_object_id id;
1921
1922 if (asprintf(&path, "%s/%s", path_objects, object_dir) == -1) {
1923 err = got_error_from_errno("asprintf");
1924 goto done;
1925 }
1926
1927 dir = opendir(path);
1928 if (dir == NULL) {
1929 if (errno == ENOENT) {
1930 err = NULL;
1931 goto done;
1932 }
1933 err = got_error_from_errno2("opendir", path);
1934 goto done;
1935 }
1936 while ((dent = readdir(dir)) != NULL) {
1937 int cmp;
1938
1939 free(id_str);
1940 id_str = NULL;
1941
1942 if (strcmp(dent->d_name, ".") == 0 ||
1943 strcmp(dent->d_name, "..") == 0)
1944 continue;
1945
1946 if (asprintf(&id_str, "%s%s", object_dir, dent->d_name) == -1) {
1947 err = got_error_from_errno("asprintf");
1948 goto done;
1949 }
1950
1951 if (!got_parse_object_id(&id, id_str, repo->algo))
1952 continue;
1953
1954 /*
1955 * Directory entries do not necessarily appear in
1956 * sorted order, so we must iterate over all of them.
1957 */
1958 cmp = strncmp(id_str, id_str_prefix, strlen(id_str_prefix));
1959 if (cmp != 0)
1960 continue;
1961
1962 if (*unique_id == NULL) {
1963 if (obj_type != GOT_OBJ_TYPE_ANY) {
1964 int matched_type;
1965 err = got_object_get_type(&matched_type, repo,
1966 &id);
1967 if (err)
1968 goto done;
1969 if (matched_type != obj_type)
1970 continue;
1971 }
1972 *unique_id = got_object_id_dup(&id);
1973 if (*unique_id == NULL) {
1974 err = got_error_from_errno("got_object_id_dup");
1975 goto done;
1976 }
1977 } else {
1978 if (got_object_id_cmp(*unique_id, &id) == 0)
1979 continue; /* both packed and loose */
1980 err = got_error(GOT_ERR_AMBIGUOUS_ID);
1981 goto done;
1982 }
1983 }
1984 done:
1985 if (dir && closedir(dir) != 0 && err == NULL)
1986 err = got_error_from_errno("closedir");
1987 if (err) {
1988 free(*unique_id);
1989 *unique_id = NULL;
1990 }
1991 free(id_str);
1992 free(path);
1993 return err;
1994 }
1995
1996 const struct got_error *
1997 got_repo_match_object_id_prefix(struct got_object_id **id,
1998 const char *id_str_prefix, int obj_type, struct got_repository *repo)
1999 {
2000 const struct got_error *err = NULL;
2001 char *path_objects = NULL, *object_dir = NULL;
2002 size_t len, digest_string_len;
2003 int i;
2004
2005 *id = NULL;
2006
2007 path_objects = got_repo_get_path_objects(repo);
2008 digest_string_len = got_hash_digest_string_length(repo->algo);
2009
2010 len = strlen(id_str_prefix);
2011 if (len > digest_string_len - 1) {
2012 err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
2013 goto done;
2014 }
2015
2016 for (i = 0; i < len; i++) {
2017 if (isxdigit((unsigned char)id_str_prefix[i]))
2018 continue;
2019 err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
2020 goto done;
2021 }
2022
2023 if (len >= 2) {
2024 err = match_packed_object(id, repo, id_str_prefix, obj_type);
2025 if (err)
2026 goto done;
2027 object_dir = strndup(id_str_prefix, 2);
2028 if (object_dir == NULL) {
2029 err = got_error_from_errno("strdup");
2030 goto done;
2031 }
2032 err = match_loose_object(id, path_objects, object_dir,
2033 id_str_prefix, obj_type, repo);
2034 } else if (len == 1) {
2035 int i;
2036 for (i = 0; i < 0xf; i++) {
2037 if (asprintf(&object_dir, "%s%.1x", id_str_prefix, i)
2038 == -1) {
2039 err = got_error_from_errno("asprintf");
2040 goto done;
2041 }
2042 err = match_packed_object(id, repo, object_dir,
2043 obj_type);
2044 if (err)
2045 goto done;
2046 err = match_loose_object(id, path_objects, object_dir,
2047 id_str_prefix, obj_type, repo);
2048 if (err)
2049 goto done;
2050 }
2051 } else {
2052 err = got_error_path(id_str_prefix, GOT_ERR_BAD_OBJ_ID_STR);
2053 goto done;
2054 }
2055 done:
2056 free(path_objects);
2057 free(object_dir);
2058 if (err) {
2059 free(*id);
2060 *id = NULL;
2061 } else if (*id == NULL) {
2062 switch (obj_type) {
2063 case GOT_OBJ_TYPE_BLOB:
2064 err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
2065 GOT_OBJ_LABEL_BLOB, id_str_prefix);
2066 break;
2067 case GOT_OBJ_TYPE_TREE:
2068 err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
2069 GOT_OBJ_LABEL_TREE, id_str_prefix);
2070 break;
2071 case GOT_OBJ_TYPE_COMMIT:
2072 err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
2073 GOT_OBJ_LABEL_COMMIT, id_str_prefix);
2074 break;
2075 case GOT_OBJ_TYPE_TAG:
2076 err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
2077 GOT_OBJ_LABEL_TAG, id_str_prefix);
2078 break;
2079 default:
2080 err = got_error_path(id_str_prefix, GOT_ERR_NO_OBJ);
2081 break;
2082 }
2083 }
2084
2085 return err;
2086 }
2087
2088 const struct got_error *
2089 got_repo_match_object_id(struct got_object_id **id, char **label,
2090 const char *id_str, int obj_type, struct got_reflist_head *refs,
2091 struct got_repository *repo)
2092 {
2093 const struct got_error *err;
2094 struct got_tag_object *tag;
2095 struct got_reference *ref = NULL;
2096
2097 *id = NULL;
2098 if (label)
2099 *label = NULL;
2100
2101 if (refs) {
2102 err = got_repo_object_match_tag(&tag, id_str, obj_type,
2103 refs, repo);
2104 if (err == NULL) {
2105 *id = got_object_id_dup(
2106 got_object_tag_get_object_id(tag));
2107 if (*id == NULL)
2108 err = got_error_from_errno("got_object_id_dup");
2109 else if (label && asprintf(label, "refs/tags/%s",
2110 got_object_tag_get_name(tag)) == -1) {
2111 err = got_error_from_errno("asprintf");
2112 free(*id);
2113 *id = NULL;
2114 }
2115 got_object_tag_close(tag);
2116 return err;
2117 } else if (err->code != GOT_ERR_OBJ_TYPE &&
2118 err->code != GOT_ERR_NO_OBJ)
2119 return err;
2120 }
2121
2122 err = got_ref_open(&ref, repo, id_str, 0);
2123 if (err == NULL) {
2124 err = got_ref_resolve(id, repo, ref);
2125 if (err)
2126 goto done;
2127 if (label) {
2128 *label = strdup(got_ref_get_name(ref));
2129 if (*label == NULL) {
2130 err = got_error_from_errno("strdup");
2131 goto done;
2132 }
2133 }
2134 } else {
2135 if (err->code != GOT_ERR_NOT_REF &&
2136 err->code != GOT_ERR_BAD_REF_NAME)
2137 goto done;
2138 err = got_repo_match_object_id_prefix(id, id_str,
2139 obj_type, repo);
2140 if (err) {
2141 if (err->code == GOT_ERR_BAD_OBJ_ID_STR)
2142 err = got_error_not_ref(id_str);
2143 goto done;
2144 }
2145 if (label) {
2146 err = got_object_id_str(label, *id);
2147 if (*label == NULL) {
2148 err = got_error_from_errno("strdup");
2149 goto done;
2150 }
2151 }
2152 }
2153 done:
2154 if (ref)
2155 got_ref_close(ref);
2156 return err;
2157 }
2158
2159 const struct got_error *
2160 got_repo_object_match_tag(struct got_tag_object **tag, const char *name,
2161 int obj_type, struct got_reflist_head *refs, struct got_repository *repo)
2162 {
2163 const struct got_error *err = NULL;
2164 struct got_reflist_entry *re;
2165 struct got_object_id *tag_id;
2166 int name_is_absolute = (strncmp(name, "refs/", 5) == 0);
2167
2168 *tag = NULL;
2169
2170 TAILQ_FOREACH(re, refs, entry) {
2171 const char *refname;
2172 refname = got_ref_get_name(re->ref);
2173 if (got_ref_is_symbolic(re->ref))
2174 continue;
2175 if (strncmp(refname, "refs/tags/", 10) != 0)
2176 continue;
2177 if (!name_is_absolute)
2178 refname += strlen("refs/tags/");
2179 if (strcmp(refname, name) != 0)
2180 continue;
2181 err = got_ref_resolve(&tag_id, repo, re->ref);
2182 if (err)
2183 break;
2184 err = got_object_open_as_tag(tag, repo, tag_id);
2185 free(tag_id);
2186 if (err)
2187 break;
2188 if (obj_type == GOT_OBJ_TYPE_ANY ||
2189 got_object_tag_get_object_type(*tag) == obj_type)
2190 break;
2191 got_object_tag_close(*tag);
2192 *tag = NULL;
2193 }
2194
2195 if (err == NULL && *tag == NULL)
2196 err = got_error_fmt(GOT_ERR_NO_OBJ, "%s %s",
2197 GOT_OBJ_LABEL_TAG, name);
2198 return err;
2199 }
2200
2201 const struct got_error *
2202 got_repo_find_object_id(struct got_object_id *id, struct got_repository *repo)
2203 {
2204 const struct got_error *err;
2205 struct got_object_id *matched_id = NULL;
2206 char *id_str = NULL;
2207
2208 err = got_object_id_str(&id_str, id);
2209 if (err)
2210 return err;
2211
2212 err = got_repo_match_object_id_prefix(&matched_id, id_str,
2213 GOT_OBJ_TYPE_ANY, repo);
2214 free(matched_id);
2215 free(id_str);
2216 return err;
2217 }
2218
2219 static const struct got_error *
2220 alloc_added_blob_tree_entry(struct got_tree_entry **new_te,
2221 const char *name, mode_t mode, struct got_object_id *blob_id)
2222 {
2223 const struct got_error *err = NULL;
2224
2225 *new_te = NULL;
2226
2227 *new_te = calloc(1, sizeof(**new_te));
2228 if (*new_te == NULL)
2229 return got_error_from_errno("calloc");
2230
2231 if (strlcpy((*new_te)->name, name, sizeof((*new_te)->name)) >=
2232 sizeof((*new_te)->name)) {
2233 err = got_error(GOT_ERR_NO_SPACE);
2234 goto done;
2235 }
2236
2237 if (S_ISLNK(mode)) {
2238 (*new_te)->mode = S_IFLNK;
2239 } else {
2240 (*new_te)->mode = S_IFREG;
2241 (*new_te)->mode |= (mode & (S_IRWXU | S_IRWXG | S_IRWXO));
2242 }
2243 memcpy(&(*new_te)->id, blob_id, sizeof((*new_te)->id));
2244 done:
2245 if (err && *new_te) {
2246 free(*new_te);
2247 *new_te = NULL;
2248 }
2249 return err;
2250 }
2251
2252 static const struct got_error *
2253 import_file(struct got_tree_entry **new_te, struct dirent *de,
2254 const char *path, struct got_repository *repo)
2255 {
2256 const struct got_error *err;
2257 struct got_object_id *blob_id = NULL;
2258 char *filepath;
2259 struct stat sb;
2260
2261 if (asprintf(&filepath, "%s%s%s", path,
2262 path[0] == '\0' ? "" : "/", de->d_name) == -1)
2263 return got_error_from_errno("asprintf");
2264
2265 if (lstat(filepath, &sb) != 0) {
2266 err = got_error_from_errno2("lstat", path);
2267 goto done;
2268 }
2269
2270 err = got_object_blob_create(&blob_id, filepath, repo);
2271 if (err)
2272 goto done;
2273
2274 err = alloc_added_blob_tree_entry(new_te, de->d_name, sb.st_mode,
2275 blob_id);
2276 done:
2277 free(filepath);
2278 if (err)
2279 free(blob_id);
2280 return err;
2281 }
2282
2283 static const struct got_error *
2284 insert_dir_entry(struct dirent *de, long pos,
2285 struct got_pathlist_head *dirents)
2286 {
2287 const struct got_error *err = NULL;
2288 struct got_pathlist_entry *new_pe;
2289 char *d_name;
2290 long *ppos;
2291
2292 d_name = strdup(de->d_name);
2293 if (d_name == NULL)
2294 return got_error_from_errno("strdup");
2295
2296 /* Record the offset to this dirent in struct DIR as path data. */
2297 ppos = malloc(sizeof(*ppos));
2298 if (ppos == NULL) {
2299 err = got_error_from_errno("malloc");
2300 goto done;
2301 }
2302 *ppos = pos;
2303
2304 err = got_pathlist_insert(&new_pe, dirents, d_name, ppos);
2305 if (err)
2306 goto done;
2307
2308 if (new_pe == NULL)
2309 err = got_error(GOT_ERR_TREE_DUP_ENTRY);
2310 done:
2311 if (err) {
2312 free(d_name);
2313 free(ppos);
2314 }
2315 return err;
2316 }
2317
2318 static const struct got_error *
2319 insert_tree_entry(struct got_tree_entry *new_te,
2320 struct got_pathlist_head *paths)
2321 {
2322 const struct got_error *err = NULL;
2323 struct got_pathlist_entry *new_pe;
2324
2325 err = got_pathlist_insert(&new_pe, paths, new_te->name, new_te);
2326 if (err)
2327 return err;
2328 if (new_pe == NULL)
2329 return got_error(GOT_ERR_TREE_DUP_ENTRY);
2330 return NULL;
2331 }
2332
2333 static const struct got_error *write_tree(struct got_object_id **,
2334 const char *, struct got_pathlist_head *, struct got_repository *,
2335 got_repo_import_cb progress_cb, void *progress_arg);
2336
2337 static const struct got_error *
2338 import_subdir(struct got_tree_entry **new_te, struct dirent *de,
2339 const char *path, struct got_pathlist_head *ignores,
2340 struct got_repository *repo,
2341 got_repo_import_cb progress_cb, void *progress_arg)
2342 {
2343 const struct got_error *err;
2344 struct got_object_id *id = NULL;
2345 char *subdirpath;
2346
2347 if (asprintf(&subdirpath, "%s%s%s", path,
2348 path[0] == '\0' ? "" : "/", de->d_name) == -1)
2349 return got_error_from_errno("asprintf");
2350
2351 (*new_te) = calloc(1, sizeof(**new_te));
2352 if (*new_te == NULL)
2353 return got_error_from_errno("calloc");
2354 (*new_te)->mode = S_IFDIR;
2355 if (strlcpy((*new_te)->name, de->d_name, sizeof((*new_te)->name)) >=
2356 sizeof((*new_te)->name)) {
2357 err = got_error(GOT_ERR_NO_SPACE);
2358 goto done;
2359 }
2360 err = write_tree(&id, subdirpath, ignores, repo,
2361 progress_cb, progress_arg);
2362 if (err)
2363 goto done;
2364 memcpy(&(*new_te)->id, id, sizeof((*new_te)->id));
2365
2366 done:
2367 free(id);
2368 free(subdirpath);
2369 if (err) {
2370 free(*new_te);
2371 *new_te = NULL;
2372 }
2373 return err;
2374 }
2375
2376 static const struct got_error *
2377 match_ignores(int *ignore, char *d_name, int type,
2378 struct got_pathlist_head *ignores)
2379 {
2380 struct got_pathlist_entry *pe;
2381
2382 *ignore = 0;
2383
2384 RB_FOREACH(pe, got_pathlist_head, ignores) {
2385 if (type == DT_DIR && pe->path_len > 0 &&
2386 pe->path[pe->path_len - 1] == '/') {
2387 char stripped[PATH_MAX];
2388
2389 if (strlcpy(stripped, pe->path,
2390 sizeof(stripped)) >= sizeof(stripped))
2391 return got_error(GOT_ERR_NO_SPACE);
2392
2393 got_path_strip_trailing_slashes(stripped);
2394 if (fnmatch(stripped, d_name, 0) == 0) {
2395 *ignore = 1;
2396 break;
2397 }
2398 } else if (fnmatch(pe->path, d_name, 0) == 0) {
2399 *ignore = 1;
2400 break;
2401 }
2402 }
2403
2404 return NULL;
2405 }
2406
2407 static const struct got_error *
2408 write_tree(struct got_object_id **new_tree_id, const char *path_dir,
2409 struct got_pathlist_head *ignores, struct got_repository *repo,
2410 got_repo_import_cb progress_cb, void *progress_arg)
2411 {
2412 const struct got_error *err = NULL;
2413 DIR *dir;
2414 struct dirent *de;
2415 int nentries;
2416 struct got_tree_entry *new_te = NULL;
2417 struct got_pathlist_head paths, dirents;
2418 struct got_pathlist_entry *pe;
2419 long pos;
2420
2421 *new_tree_id = NULL;
2422
2423 RB_INIT(&paths);
2424 RB_INIT(&dirents);
2425
2426 dir = opendir(path_dir);
2427 if (dir == NULL) {
2428 err = got_error_from_errno2("opendir", path_dir);
2429 goto done;
2430 }
2431
2432 /*
2433 * Sort dirents into a pathlist. Otherwise subdir-iteration order
2434 * depends on the readdir() function, which varies between different
2435 * operating systems, making regression testing more difficult.
2436 * This smells of TOCTOU, but if dirents appear, disappear, or change
2437 * during import then we have a racy situation no matter what.
2438 */
2439 pos = telldir(dir);
2440 if (pos == -1) {
2441 err = got_error_from_errno2("telldir", path_dir);
2442 goto done;
2443 }
2444 while ((de = readdir(dir)) != NULL) {
2445 if (strcmp(de->d_name, ".") != 0 &&
2446 strcmp(de->d_name, "..") != 0) {
2447 err = insert_dir_entry(de, pos, &dirents);
2448 if (err)
2449 goto done;
2450 }
2451 pos = telldir(dir);
2452 if (pos == -1) {
2453 err = got_error_from_errno2("telldir", path_dir);
2454 goto done;
2455 }
2456 }
2457
2458 nentries = 0;
2459 RB_FOREACH(pe, got_pathlist_head, &dirents) {
2460 int ignore = 0;
2461 int type;
2462 long *pos = pe->data;
2463
2464 seekdir(dir, *pos);
2465 de = readdir(dir);
2466 if (de == NULL) { /* should not happen */
2467 err = got_error_fmt(GOT_ERR_EOF,
2468 "unexpected EOF on directory '%s' while seeking "
2469 "to entry '%s' at offset '%ld'\n", path_dir,
2470 pe->path, *pos);
2471 goto done;
2472 }
2473
2474 err = got_path_dirent_type(&type, path_dir, de);
2475 if (err)
2476 goto done;
2477
2478 err = match_ignores(&ignore, de->d_name, type, ignores);
2479 if (ignore)
2480 continue;
2481
2482 if (type == DT_DIR) {
2483 err = import_subdir(&new_te, de, path_dir,
2484 ignores, repo, progress_cb, progress_arg);
2485 if (err) {
2486 if (err->code != GOT_ERR_NO_TREE_ENTRY)
2487 goto done;
2488 err = NULL;
2489 continue;
2490 }
2491 } else if (type == DT_REG || type == DT_LNK) {
2492 err = import_file(&new_te, de, path_dir, repo);
2493 if (err)
2494 goto done;
2495 } else
2496 continue;
2497
2498 err = insert_tree_entry(new_te, &paths);
2499 if (err)
2500 goto done;
2501 nentries++;
2502 }
2503
2504 if (RB_EMPTY(&paths)) {
2505 err = got_error_msg(GOT_ERR_NO_TREE_ENTRY,
2506 "cannot create tree without any entries");
2507 goto done;
2508 }
2509
2510 RB_FOREACH(pe, got_pathlist_head, &paths) {
2511 struct got_tree_entry *te = pe->data;
2512 char *path;
2513 if (!S_ISREG(te->mode) && !S_ISLNK(te->mode))
2514 continue;
2515 if (asprintf(&path, "%s/%s", path_dir, pe->path) == -1) {
2516 err = got_error_from_errno("asprintf");
2517 goto done;
2518 }
2519 err = (*progress_cb)(progress_arg, path);
2520 free(path);
2521 if (err)
2522 goto done;
2523 }
2524
2525 err = got_object_tree_create(new_tree_id, &paths, nentries, repo);
2526 done:
2527 if (dir)
2528 closedir(dir);
2529 got_pathlist_free(&dirents, GOT_PATHLIST_FREE_ALL);
2530 got_pathlist_free(&paths, GOT_PATHLIST_FREE_NONE);
2531 return err;
2532 }
2533
2534 const struct got_error *
2535 got_repo_import(struct got_object_id **new_commit_id, const char *path_dir,
2536 const char *logmsg, const char *author, struct got_pathlist_head *ignores,
2537 struct got_repository *repo, got_repo_import_cb progress_cb,
2538 void *progress_arg)
2539 {
2540 const struct got_error *err;
2541 struct got_object_id *new_tree_id;
2542
2543 err = write_tree(&new_tree_id, path_dir, ignores, repo,
2544 progress_cb, progress_arg);
2545 if (err)
2546 return err;
2547
2548 err = got_object_commit_create(new_commit_id, new_tree_id, NULL, 0,
2549 author, time(NULL), author, time(NULL), logmsg, repo);
2550 free(new_tree_id);
2551 return err;
2552 }
2553
2554 const struct got_error *
2555 got_repo_get_loose_object_info(int *nobjects, off_t *ondisk_size,
2556 struct got_repository *repo)
2557 {
2558 const struct got_error *err = NULL;
2559 char *path_objects = NULL, *path = NULL;
2560 DIR *dir = NULL;
2561 struct got_object_id id;
2562 int i;
2563
2564 *nobjects = 0;
2565 *ondisk_size = 0;
2566
2567 path_objects = got_repo_get_path_objects(repo);
2568 if (path_objects == NULL)
2569 return got_error_from_errno("got_repo_get_path_objects");
2570
2571 for (i = 0; i <= 0xff; i++) {
2572 struct dirent *dent;
2573
2574 if (asprintf(&path, "%s/%.2x", path_objects, i) == -1) {
2575 err = got_error_from_errno("asprintf");
2576 break;
2577 }
2578
2579 dir = opendir(path);
2580 if (dir == NULL) {
2581 if (errno == ENOENT) {
2582 err = NULL;
2583 continue;
2584 }
2585 err = got_error_from_errno2("opendir", path);
2586 break;
2587 }
2588
2589 while ((dent = readdir(dir)) != NULL) {
2590 char *id_str;
2591 int fd;
2592 struct stat sb;
2593
2594 if (strcmp(dent->d_name, ".") == 0 ||
2595 strcmp(dent->d_name, "..") == 0)
2596 continue;
2597
2598 if (asprintf(&id_str, "%.2x%s", i, dent->d_name) == -1) {
2599 err = got_error_from_errno("asprintf");
2600 goto done;
2601 }
2602
2603 if (!got_parse_object_id(&id, id_str, repo->algo)) {
2604 free(id_str);
2605 continue;
2606 }
2607 free(id_str);
2608
2609 err = got_object_open_loose_fd(&fd, &id, repo);
2610 if (err)
2611 goto done;
2612
2613 if (fstat(fd, &sb) == -1) {
2614 err = got_error_from_errno("fstat");
2615 close(fd);
2616 goto done;
2617 }
2618 (*nobjects)++;
2619 (*ondisk_size) += sb.st_size;
2620
2621 if (close(fd) == -1) {
2622 err = got_error_from_errno("close");
2623 goto done;
2624 }
2625 }
2626
2627 if (closedir(dir) != 0) {
2628 err = got_error_from_errno("closedir");
2629 goto done;
2630 }
2631 dir = NULL;
2632
2633 free(path);
2634 path = NULL;
2635 }
2636 done:
2637 if (dir && closedir(dir) != 0 && err == NULL)
2638 err = got_error_from_errno("closedir");
2639
2640 if (err) {
2641 *nobjects = 0;
2642 *ondisk_size = 0;
2643 }
2644 free(path_objects);
2645 free(path);
2646 return err;
2647 }
2648
2649 const struct got_error *
2650 got_repo_get_packfile_info(int *npackfiles, int *nobjects,
2651 off_t *total_packsize, struct got_repository *repo)
2652 {
2653 const struct got_error *err = NULL;
2654 DIR *packdir = NULL;
2655 struct dirent *dent;
2656 struct got_packidx *packidx = NULL;
2657 char *path_packidx;
2658 char *path_packfile;
2659 int packdir_fd;
2660 struct stat sb;
2661
2662 *npackfiles = 0;
2663 *nobjects = 0;
2664 *total_packsize = 0;
2665
2666 packdir_fd = openat(got_repo_get_fd(repo),
2667 GOT_OBJECTS_PACK_DIR, O_DIRECTORY);
2668 if (packdir_fd == -1) {
2669 return got_error_from_errno_fmt("openat: %s/%s",
2670 got_repo_get_path_git_dir(repo),
2671 GOT_OBJECTS_PACK_DIR);
2672 }
2673
2674 packdir = fdopendir(packdir_fd);
2675 if (packdir == NULL) {
2676 err = got_error_from_errno("fdopendir");
2677 close(packdir_fd);
2678 goto done;
2679 }
2680
2681 while ((dent = readdir(packdir)) != NULL) {
2682 if (!got_repo_is_packidx_filename(dent->d_name,
2683 strlen(dent->d_name),
2684 repo->algo))
2685 continue;
2686
2687 if (asprintf(&path_packidx, "%s/%s", GOT_OBJECTS_PACK_DIR,
2688 dent->d_name) == -1) {
2689 err = got_error_from_errno("asprintf");
2690 goto done;
2691 }
2692
2693 err = got_packidx_open(&packidx, got_repo_get_fd(repo),
2694 path_packidx, 0, repo->algo);
2695 free(path_packidx);
2696 if (err) {
2697 if (err->code == GOT_ERR_LONELY_PACKIDX) {
2698 err = NULL;
2699 continue;
2700 }
2701 goto done;
2702 }
2703
2704 if (fstat(packidx->fd, &sb) == -1)
2705 goto done;
2706 *total_packsize += sb.st_size;
2707
2708 err = got_packidx_get_packfile_path(&path_packfile,
2709 packidx->path_packidx);
2710 if (err)
2711 goto done;
2712
2713 if (fstatat(got_repo_get_fd(repo), path_packfile, &sb,
2714 0) == -1) {
2715 free(path_packfile);
2716 goto done;
2717 }
2718 free(path_packfile);
2719 *total_packsize += sb.st_size;
2720
2721 *nobjects += be32toh(packidx->hdr.fanout_table[0xff]);
2722
2723 (*npackfiles)++;
2724
2725 got_packidx_close(packidx);
2726 packidx = NULL;
2727 }
2728 done:
2729 if (packidx)
2730 got_packidx_close(packidx);
2731 if (packdir && closedir(packdir) != 0 && err == NULL)
2732 err = got_error_from_errno("closedir");
2733 if (err) {
2734 *npackfiles = 0;
2735 *nobjects = 0;
2736 *total_packsize = 0;
2737 }
2738 return err;
2739 }
2740
2741 RB_GENERATE(got_packidx_bloom_filter_tree, got_packidx_bloom_filter, entry,
2742 got_packidx_bloom_filter_cmp);
portable version of Game of Trees