Btrfs progs v4.17.1
[btrfs-progs-unstable/devel.git] / tests / fssum.c
blob2bda5df875268ac8c233488ce29f4a4900eca676
1 /*
2 * Copyright (C) 2012 STRATO AG. All rights reserved.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public
6 * License v2 as published by the Free Software Foundation.
8 * This program is distributed in the hope that it will be useful,
9 * but WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
13 * You should have received a copy of the GNU General Public
14 * License along with this program; if not, write to the
15 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
16 * Boston, MA 021110-1307, USA.
19 #include "kerncompat.h"
20 #include <stdio.h>
21 #include <stdlib.h>
22 #include <unistd.h>
23 #include <string.h>
24 #include <fcntl.h>
25 #include <dirent.h>
26 #include <errno.h>
27 #include <sys/sysmacros.h>
28 #include <sys/stat.h>
29 #include <assert.h>
30 #include <time.h>
31 #include <stdint.h>
32 #include "tests/sha.h"
34 #define CS_SIZE 32
35 #define CHUNKS 128
37 #ifndef SEEK_DATA
38 #define SEEK_DATA 3
39 #define SEEK_HOLE 4
40 #endif
42 /* TODO: add hardlink recognition */
43 /* TODO: add xattr/acl */
45 struct excludes {
46 char *path;
47 int len;
50 typedef struct _sum {
51 SHA256Context sha;
52 unsigned char out[CS_SIZE];
53 } sum_t;
55 typedef int (*sum_file_data_t)(int fd, sum_t *dst);
57 int gen_manifest = 0;
58 int in_manifest = 0;
59 char *checksum = NULL;
60 struct excludes *excludes;
61 int n_excludes = 0;
62 int verbose = 0;
63 FILE *out_fp;
64 FILE *in_fp;
66 enum _flags {
67 FLAG_UID,
68 FLAG_GID,
69 FLAG_MODE,
70 FLAG_ATIME,
71 FLAG_MTIME,
72 FLAG_CTIME,
73 FLAG_DATA,
74 FLAG_OPEN_ERROR,
75 FLAG_STRUCTURE,
76 NUM_FLAGS
79 const char flchar[] = "ugoamcdes";
80 char line[65536];
82 int flags[NUM_FLAGS] = {1, 1, 1, 1, 1, 0, 1, 0, 0};
84 char *
85 getln(char *buf, int size, FILE *fp)
87 char *p;
88 int l;
90 p = fgets(buf, size, fp);
91 if (!p)
92 return NULL;
94 l = strlen(p);
95 while(l > 0 && (p[l - 1] == '\n' || p[l - 1] == '\r'))
96 p[--l] = 0;
98 return p;
101 void
102 parse_flag(int c)
104 int i;
105 int is_upper = 0;
107 if (c >= 'A' && c <= 'Z') {
108 is_upper = 1;
109 c += 'a' - 'A';
111 for (i = 0; flchar[i]; ++i) {
112 if (flchar[i] == c) {
113 flags[i] = is_upper ? 0 : 1;
114 return;
117 fprintf(stderr, "unrecognized flag %c\n", c);
118 exit(-1);
121 void
122 parse_flags(char *p)
124 while (*p)
125 parse_flag(*p++);
128 void
129 usage(void)
131 fprintf(stderr, "usage: fssum <options> <path>\n");
132 fprintf(stderr, " options:\n");
133 fprintf(stderr, " -f : write out a full manifest file\n");
134 fprintf(stderr, " -w <file> : send output to file\n");
135 fprintf(stderr, " -v : verbose mode (debugging only)\n");
136 fprintf(stderr,
137 " -r <file> : read checksum or manifest from file\n");
138 fprintf(stderr, " -[ugoamcde] : specify which fields to include in checksum calculation.\n");
139 fprintf(stderr, " u : include uid\n");
140 fprintf(stderr, " g : include gid\n");
141 fprintf(stderr, " o : include mode\n");
142 fprintf(stderr, " m : include mtime\n");
143 fprintf(stderr, " a : include atime\n");
144 fprintf(stderr, " c : include ctime\n");
145 fprintf(stderr, " d : include file data\n");
146 fprintf(stderr, " e : include open errors (aborts otherwise)\n");
147 fprintf(stderr, " s : include block structure (holes)\n");
148 fprintf(stderr, " -[UGOAMCDES]: exclude respective field from calculation\n");
149 fprintf(stderr, " -n : reset all flags\n");
150 fprintf(stderr, " -N : set all flags\n");
151 fprintf(stderr, " -x path : exclude path when building checksum (multiple ok)\n");
152 fprintf(stderr, " -h : this help\n\n");
153 fprintf(stderr, "The default field mask is ugoamCdES. If the checksum/manifest is read from a\n");
154 fprintf(stderr, "file, the mask is taken from there and the values given on the command line\n");
155 fprintf(stderr, "are ignored.\n");
156 exit(-1);
159 static char buf[65536];
161 void *
162 alloc(size_t sz)
164 void *p = malloc(sz);
166 if (!p) {
167 fprintf(stderr, "malloc failed\n");
168 exit(-1);
171 return p;
174 void
175 sum_init(sum_t *cs)
177 SHA256Reset(&cs->sha);
180 void
181 sum_fini(sum_t *cs)
183 SHA256Result(&cs->sha, cs->out);
186 void
187 sum_add(sum_t *cs, void *buf, int size)
189 SHA256Input(&cs->sha, buf, size);
192 void
193 sum_add_sum(sum_t *dst, sum_t *src)
195 sum_add(dst, src->out, sizeof(src->out));
198 void
199 sum_add_u64(sum_t *dst, uint64_t val)
201 uint64_t v = cpu_to_le64(val);
202 sum_add(dst, &v, sizeof(v));
205 void
206 sum_add_time(sum_t *dst, time_t t)
208 sum_add_u64(dst, t);
211 char *
212 sum_to_string(sum_t *dst)
214 int i;
215 char *s = alloc(CS_SIZE * 2 + 1);
217 for (i = 0; i < CS_SIZE; ++i)
218 sprintf(s + i * 2, "%02x", dst->out[i]);
220 return s;
224 sum_file_data_permissive(int fd, sum_t *dst)
226 int ret;
227 off_t pos;
228 off_t old;
229 int i;
230 uint64_t zeros = 0;
232 pos = lseek(fd, 0, SEEK_CUR);
233 if (pos == (off_t)-1)
234 return errno == ENXIO ? 0 : -2;
236 while (1) {
237 old = pos;
238 pos = lseek(fd, pos, SEEK_DATA);
239 if (pos == (off_t)-1) {
240 if (errno == ENXIO) {
241 ret = 0;
242 pos = lseek(fd, 0, SEEK_END);
243 if (pos != (off_t)-1)
244 zeros += pos - old;
245 } else {
246 ret = -2;
248 break;
250 ret = read(fd, buf, sizeof(buf));
251 assert(ret); /* eof found by lseek */
252 if (ret <= 0)
253 break;
254 if (old < pos) /* hole */
255 zeros += pos - old;
256 for (i = 0; i < ret; ++i) {
257 for (old = i; buf[i] == 0 && i < ret; ++i)
259 if (old < i) /* code like a hole */
260 zeros += i - old;
261 if (i == ret)
262 break;
263 if (zeros) {
264 if (verbose >= 2)
265 fprintf(stderr,
266 "adding %llu zeros to sum\n",
267 (unsigned long long)zeros);
268 sum_add_u64(dst, 0);
269 sum_add_u64(dst, zeros);
270 zeros = 0;
272 for (old = i; buf[i] != 0 && i < ret; ++i)
274 if (verbose >= 2)
275 fprintf(stderr, "adding %u non-zeros to sum\n",
276 i - (int)old);
277 sum_add(dst, buf + old, i - old);
279 pos += ret;
282 if (zeros) {
283 if (verbose >= 2)
284 fprintf(stderr,
285 "adding %llu zeros to sum (finishing)\n",
286 (unsigned long long)zeros);
287 sum_add_u64(dst, 0);
288 sum_add_u64(dst, zeros);
291 return ret;
295 sum_file_data_strict(int fd, sum_t *dst)
297 int ret;
298 off_t pos;
300 pos = lseek(fd, 0, SEEK_CUR);
301 if (pos == (off_t)-1)
302 return errno == ENXIO ? 0 : -2;
304 while (1) {
305 pos = lseek(fd, pos, SEEK_DATA);
306 if (pos == (off_t)-1)
307 return errno == ENXIO ? 0 : -2;
308 ret = read(fd, buf, sizeof(buf));
309 assert(ret); /* eof found by lseek */
310 if (ret <= 0)
311 return ret;
312 if (verbose >= 2)
313 fprintf(stderr,
314 "adding to sum at file offset %llu, %d bytes\n",
315 (unsigned long long)pos, ret);
316 sum_add_u64(dst, (uint64_t)pos);
317 sum_add(dst, buf, ret);
318 pos += ret;
322 char *
323 escape(char *in)
325 char *out = alloc(strlen(in) * 3 + 1);
326 char *src = in;
327 char *dst = out;
329 for (; *src; ++src) {
330 if (*src >= 32 && *src < 127 && *src != '\\') {
331 *dst++ = *src;
332 } else {
333 sprintf(dst, "\\%02x", (unsigned char)*src);
334 dst += 3;
337 *dst = 0;
339 return out;
342 void
343 excess_file(const char *fn)
345 printf("only in local fs: %s\n", fn);
348 void
349 missing_file(const char *fn)
351 printf("only in remote fs: %s\n", fn);
355 pathcmp(const char *a, const char *b)
357 int len_a = strlen(a);
358 int len_b = strlen(b);
361 * as the containing directory is sent after the files, it has to
362 * come out bigger in the comparison.
364 if (len_a < len_b && a[len_a - 1] == '/' && strncmp(a, b, len_a) == 0)
365 return 1;
366 if (len_a > len_b && b[len_b - 1] == '/' && strncmp(a, b, len_b) == 0)
367 return -1;
369 return strcmp(a, b);
372 void
373 check_match(char *fn, char *local_m, char *remote_m,
374 char *local_c, char *remote_c)
376 int match_m = !strcmp(local_m, remote_m);
377 int match_c = !strcmp(local_c, remote_c);
379 if (match_m && !match_c) {
380 printf("data mismatch in %s\n", fn);
381 } else if (!match_m && match_c) {
382 printf("metadata mismatch in %s\n", fn);
383 } else if (!match_m && !match_c) {
384 printf("metadata and data mismatch in %s\n", fn);
388 char *prev_fn;
389 char *prev_m;
390 char *prev_c;
391 void
392 check_manifest(char *fn, char *m, char *c, int last_call)
394 char *rem_m;
395 char *rem_c;
396 char *l;
397 int cmp;
399 if (prev_fn) {
400 if (last_call)
401 cmp = -1;
402 else
403 cmp = pathcmp(prev_fn, fn);
404 if (cmp > 0) {
405 excess_file(fn);
406 return;
407 } else if (cmp < 0) {
408 missing_file(prev_fn);
409 } else {
410 check_match(fn, m, prev_m, c, prev_c);
412 free(prev_fn);
413 free(prev_m);
414 free(prev_c);
415 prev_fn = NULL;
416 prev_m = NULL;
417 prev_c = NULL;
418 if (cmp == 0)
419 return;
421 while ((l = getln(line, sizeof(line), in_fp))) {
422 rem_c = strrchr(l, ' ');
423 if (!rem_c) {
424 if (checksum)
425 free(checksum);
427 /* final cs */
428 checksum = strdup(l);
429 break;
431 if (rem_c == l) {
432 malformed:
433 fprintf(stderr, "malformed input\n");
434 exit(-1);
436 *rem_c++ = 0;
437 rem_m = strrchr(l, ' ');
438 if (!rem_m)
439 goto malformed;
440 *rem_m++ = 0;
442 if (last_call)
443 cmp = -1;
444 else
445 cmp = pathcmp(l, fn);
446 if (cmp == 0) {
447 check_match(fn, m, rem_m, c, rem_c);
448 return;
449 } else if (cmp > 0) {
450 excess_file(fn);
451 prev_fn = strdup(l);
452 prev_m = strdup(rem_m);
453 prev_c = strdup(rem_c);
454 return;
456 missing_file(l);
458 if (!last_call)
459 excess_file(fn);
463 namecmp(const void *aa, const void *bb)
465 char * const *a = aa;
466 char * const *b = bb;
468 return strcmp(*a, *b);
471 void
472 sum(int dirfd, int level, sum_t *dircs, char *path_prefix, char *path_in)
474 DIR *d;
475 struct dirent *de;
476 char **namelist = NULL;
477 int alloclen = 0;
478 int entries = 0;
479 int i;
480 int ret;
481 int fd;
482 int excl;
483 sum_file_data_t sum_file_data = flags[FLAG_STRUCTURE] ?
484 sum_file_data_strict : sum_file_data_permissive;
486 d = fdopendir(dirfd);
487 if (!d) {
488 perror("opendir");
489 exit(-1);
491 while((de = readdir(d))) {
492 if (!strcmp(de->d_name, ".") || !strcmp(de->d_name, ".."))
493 continue;
494 if (entries == alloclen) {
495 alloclen += CHUNKS;
496 namelist = realloc(namelist,
497 alloclen * sizeof(*namelist));
498 if (!namelist) {
499 fprintf(stderr, "malloc failed\n");
500 exit(-1);
503 namelist[entries] = strdup(de->d_name);
504 if (!namelist[entries]) {
505 fprintf(stderr, "malloc failed\n");
506 exit(-1);
508 ++entries;
511 qsort(namelist, entries, sizeof(*namelist), namecmp);
512 for (i = 0; i < entries; ++i) {
513 struct stat64 st;
514 sum_t cs;
515 sum_t meta;
516 char *path;
518 sum_init(&cs);
519 sum_init(&meta);
520 path = alloc(strlen(path_in) + strlen(namelist[i]) + 3);
521 sprintf(path, "%s/%s", path_in, namelist[i]);
522 for (excl = 0; excl < n_excludes; ++excl) {
523 if (strncmp(excludes[excl].path, path,
524 excludes[excl].len) == 0)
525 goto next;
528 ret = fchdir(dirfd);
529 if (ret == -1) {
530 perror("fchdir");
531 exit(-1);
533 ret = lstat64(namelist[i], &st);
534 if (ret) {
535 fprintf(stderr, "stat failed for %s/%s: %m\n",
536 path_prefix, path);
537 exit(-1);
539 sum_add_u64(&meta, level);
540 sum_add(&meta, namelist[i], strlen(namelist[i]));
541 if (!S_ISDIR(st.st_mode))
542 sum_add_u64(&meta, st.st_nlink);
543 if (flags[FLAG_UID])
544 sum_add_u64(&meta, st.st_uid);
545 if (flags[FLAG_GID])
546 sum_add_u64(&meta, st.st_gid);
547 if (flags[FLAG_MODE])
548 sum_add_u64(&meta, st.st_mode);
549 if (flags[FLAG_ATIME])
550 sum_add_time(&meta, st.st_atime);
551 if (flags[FLAG_MTIME])
552 sum_add_time(&meta, st.st_mtime);
553 if (flags[FLAG_CTIME])
554 sum_add_time(&meta, st.st_ctime);
555 if (S_ISDIR(st.st_mode)) {
556 fd = openat(dirfd, namelist[i], 0);
557 if (fd == -1 && flags[FLAG_OPEN_ERROR]) {
558 sum_add_u64(&meta, errno);
559 } else if (fd == -1) {
560 fprintf(stderr, "open failed for %s/%s: %m\n",
561 path_prefix, path);
562 exit(-1);
563 } else {
564 sum(fd, level + 1, &cs, path_prefix, path);
565 close(fd);
567 } else if (S_ISREG(st.st_mode)) {
568 sum_add_u64(&meta, st.st_size);
569 if (flags[FLAG_DATA]) {
570 if (verbose)
571 fprintf(stderr, "file %s\n",
572 namelist[i]);
573 fd = openat(dirfd, namelist[i], 0);
574 if (fd == -1 && flags[FLAG_OPEN_ERROR]) {
575 sum_add_u64(&meta, errno);
576 } else if (fd == -1) {
577 fprintf(stderr,
578 "open failed for %s/%s: %m\n",
579 path_prefix, path);
580 exit(-1);
582 if (fd != -1) {
583 ret = sum_file_data(fd, &cs);
584 if (ret < 0) {
585 fprintf(stderr,
586 "read failed for "
587 "%s/%s: %m\n",
588 path_prefix, path);
589 exit(-1);
591 close(fd);
594 } else if (S_ISLNK(st.st_mode)) {
595 ret = readlink(namelist[i], buf, sizeof(buf));
596 if (ret == -1) {
597 perror("readlink");
598 exit(-1);
600 sum_add(&cs, buf, ret);
601 } else if (S_ISCHR(st.st_mode) || S_ISBLK(st.st_mode)) {
602 sum_add_u64(&cs, major(st.st_rdev));
603 sum_add_u64(&cs, minor(st.st_rdev));
605 sum_fini(&cs);
606 sum_fini(&meta);
607 if (gen_manifest || in_manifest) {
608 char *fn;
609 char *m;
610 char *c;
612 if (S_ISDIR(st.st_mode))
613 strcat(path, "/");
614 fn = escape(path);
615 m = sum_to_string(&meta);
616 c = sum_to_string(&cs);
618 if (gen_manifest)
619 fprintf(out_fp, "%s %s %s\n", fn, m, c);
620 if (in_manifest)
621 check_manifest(fn, m, c, 0);
622 free(c);
623 free(m);
624 free(fn);
626 sum_add_sum(dircs, &cs);
627 sum_add_sum(dircs, &meta);
628 next:
629 free(path);
630 free(namelist[i]);
633 free(namelist);
634 closedir(d);
638 main(int argc, char *argv[])
640 extern char *optarg;
641 extern int optind;
642 int c;
643 char *path;
644 int fd;
645 sum_t cs;
646 char *sumstring;
647 char flagstring[sizeof(flchar)];
648 int ret = 0;
649 int i;
650 int plen;
651 int elen;
652 int n_flags = 0;
653 const char *allopts = "heEfuUgGoOaAmMcCdDsSnNw:r:vx:";
655 out_fp = stdout;
656 while ((c = getopt(argc, argv, allopts)) != EOF) {
657 switch(c) {
658 case 'f':
659 gen_manifest = 1;
660 break;
661 case 'u':
662 case 'U':
663 case 'g':
664 case 'G':
665 case 'o':
666 case 'O':
667 case 'a':
668 case 'A':
669 case 'm':
670 case 'M':
671 case 'c':
672 case 'C':
673 case 'd':
674 case 'D':
675 case 'e':
676 case 'E':
677 case 's':
678 case 'S':
679 ++n_flags;
680 parse_flag(c);
681 break;
682 case 'n':
683 for (i = 0; i < NUM_FLAGS; ++i)
684 flags[i] = 0;
685 break;
686 case 'N':
687 for (i = 0; i < NUM_FLAGS; ++i)
688 flags[i] = 1;
689 break;
690 case 'w':
691 out_fp = fopen(optarg, "w");
692 if (!out_fp) {
693 fprintf(stderr,
694 "failed to open output file: %m\n");
695 exit(-1);
697 break;
698 case 'r':
699 in_fp = fopen(optarg, "r");
700 if (!in_fp) {
701 fprintf(stderr,
702 "failed to open input file: %m\n");
703 exit(-1);
705 break;
706 case 'x':
707 ++n_excludes;
708 excludes = realloc(excludes,
709 sizeof(*excludes) * n_excludes);
710 if (!excludes) {
711 fprintf(stderr,
712 "failed to alloc exclude space\n");
713 exit(-1);
715 excludes[n_excludes - 1].path = optarg;
716 break;
717 case 'v':
718 ++verbose;
719 break;
720 case 'h':
721 case '?':
722 usage();
726 if (optind + 1 != argc) {
727 fprintf(stderr, "missing path\n");
728 usage();
731 if (in_fp) {
732 char *l = getln(line, sizeof(line), in_fp);
733 char *p;
735 if (l == NULL) {
736 fprintf(stderr, "failed to read line from input\n");
737 exit(-1);
739 if (strncmp(l, "Flags: ", 7) == 0) {
740 l += 7;
741 in_manifest = 1;
742 parse_flags(l);
743 } else if ((p = strchr(l, ':'))) {
744 *p++ = 0;
745 parse_flags(l);
747 if (checksum)
748 free(checksum);
749 checksum = strdup(p);
750 } else {
751 fprintf(stderr, "invalid input file format\n");
752 exit(-1);
754 if (n_flags)
755 fprintf(stderr, "warning: "
756 "command line flags ignored in -r mode\n");
758 strcpy(flagstring, flchar);
759 for (i = 0; i < NUM_FLAGS; ++i) {
760 if (flags[i] == 0)
761 flagstring[i] -= 'a' - 'A';
764 path = argv[optind];
765 plen = strlen(path);
766 if (path[plen - 1] == '/') {
767 --plen;
768 path[plen] = '\0';
771 for (i = 0; i < n_excludes; ++i) {
772 if (strncmp(path, excludes[i].path, plen) != 0)
773 fprintf(stderr,
774 "warning: exclude %s outside of path %s\n",
775 excludes[i].path, path);
776 else
777 excludes[i].path += plen;
778 elen = strlen(excludes[i].path);
779 if (excludes[i].path[elen - 1] == '/')
780 --elen;
781 excludes[i].path[elen] = '\0';
782 excludes[i].len = elen;
785 fd = open(path, O_RDONLY);
786 if (fd == -1) {
787 fprintf(stderr, "failed to open %s: %m\n", path);
788 exit(-1);
791 if (gen_manifest)
792 fprintf(out_fp, "Flags: %s\n", flagstring);
794 sum_init(&cs);
795 sum(fd, 1, &cs, path, "");
796 sum_fini(&cs);
798 close(fd);
799 if (in_manifest)
800 check_manifest("", "", "", 1);
802 if (!checksum) {
803 if (in_manifest) {
804 fprintf(stderr, "malformed input\n");
805 exit(-1);
807 if (!gen_manifest)
808 fprintf(out_fp, "%s:", flagstring);
810 sumstring = sum_to_string(&cs);
811 fprintf(out_fp, "%s\n", sumstring);
812 free(sumstring);
813 } else {
814 sumstring = sum_to_string(&cs);
815 if (strcmp(checksum, sumstring) == 0) {
816 printf("OK\n");
817 ret = 0;
818 } else {
819 printf("FAIL\n");
820 ret = 1;
823 free(checksum);
824 free(sumstring);
827 if (in_fp)
828 fclose(in_fp);
830 if (out_fp != stdout)
831 fclose(out_fp);
833 exit(ret);