gitbackup.c

   1 #define CHECKS
   2 #define DEBUG
   3 #define VERBOSE
   4 //#define DRY_RUN
   5 #define IGNORE_ROOT_GIT
   6
   7 #define MAX_ARGS 50
   8
   9
  10 #define _BSD_SOURCE
  11 #define _XOPEN_SOURCE 500
  12
  13 #include <assert.h>
  14 #include <errno.h>
  15 #include <stdio.h>
  16 #include <stdlib.h>
  17 #include <string.h>
  18 #include <sys/inotify.h>
  19 #include <sys/select.h>
  20 #include <sys/types.h>
  21 #include <sys/wait.h>
  22 #include <unistd.h>
  23
  24 #include <dirent.h>
  25
  26 /* When to commit:
  27  * - Every change
  28  * - Every N seconds / cron
  29  * - N seconds after changes 'settle'
  30  * Never wait longer than N seconds from a change to commit
  31  *
  32  * For each of N repositories...
  33  * When to push:
  34  * - Every commit
  35  * - Every N seconds / cron
  36  * - N seconds after changes 'settle'
  37  * - N seconds after commits 'settle'
  38  * Never wait longer than N seconds from a (change | commit) to push
  39  *
  40  * TODO: resume counter-acted schedule based on original start time?
  41  */
  42
  43 #ifndef PATH_MAX
  44 #       define PATH_MAX 512
  45 #endif
  46 #define BUFFER_SIZE (sizeof(struct inotify_event) + PATH_MAX) * 10
  47
  48 #ifdef DEBUG
  49 #       define LOG_DEBUG(fmt, stuff...) fprintf(stderr, fmt "\n", stuff)
  50 #else
  51 #       define LOG_DEBUG(...) 0
  52 #endif
  53
  54 #define MY_WATCH_EVENTS (IN_CREATE | IN_DELETE | IN_DELETE_SELF | IN_MODIFY | IN_MOVED_FROM | IN_MOVED_TO | IN_Q_OVERFLOW)
  55
  56 static
  57 int
  58 read_will_block(int fd) {
  59         fd_set fds;
  60         FD_ZERO(&fds);
  61         FD_SET(fd, &fds);
  62
  63         struct timeval tv;
  64         tv.tv_sec = 0;
  65         tv.tv_usec = 0;
  66
  67         return !select(fd + 1, &fds, NULL, NULL, &tv);
  68 }
  69
  70 static
  71 int
  72 my_system2(const char*fp, char *const argv[]) {
  73 #if defined(VERBOSE) || defined(DEBUG)
  74 #ifdef DEBUG
  75         printf("exec: %s\n", fp);
  76         for (int i = 1; argv[i]; ++i)
  77                 printf("  argv[%d]: %s\n", i, argv[i]);
  78 #else
  79         printf("'%s'", fp);
  80         for (int i = 1; argv[i]; ++i)
  81                 printf(" '%s'", argv[i]);
  82         puts("");
  83 #endif
  84 #endif
  85 #ifdef DRY_RUN
  86         return 0;
  87 #endif
  88
  89         {
  90         pid_t pid = fork();
  91         if (pid)
  92         {
  93                 // Parent
  94                 if (pid == -1)
  95                 {
  96                         perror("fork");
  97                         abort();
  98                         return -1;
  99                 }
 100                 int status;
 101                 waitpid(pid, &status, 0);
 102                 return WEXITSTATUS(status);
 103         }
 104         }
 105
 106         // Child
 107         int (*f_exec)(const char *, char *const argv[]);
 108         f_exec = (fp[0] == '/') ? execv : execvp;
 109
 110         f_exec(fp, argv);
 111         perror("exec");
 112         exit(127);
 113 }
 114 #define my_system(f, argv...) my_system2(f, (char *const[]){f, argv, NULL})
 115
 116 enum action {
 117         A_ADD,
 118         A_RM,
 119         A_REPLACE,
 120         A_MODIFY,
 121 };
 122
 123 struct vll {
 124         char*p;
 125         enum action action;
 126         struct vll*next;
 127         uint32_t flags;
 128 };
 129
 130 static
 131 void
 132 vll_set_action(struct vll**top, const char*p, enum action action, uint32_t flags) {
 133         for (struct vll *this, **thisp = top; this = *thisp; thisp = &this->next)
 134         {
 135                 if (!strcmp(this->p, p))
 136                 {
 137 #ifdef DEBUG
 138                         enum action actIn = action;
 139 #endif
 140                         switch ((this->action << 8) | action) {
 141                         case (A_RM      << 8) + A_ADD:
 142                                 action = A_REPLACE;
 143                                 break;
 144                         case (A_ADD     << 8) + A_RM:
 145                                 // Delete the plan to add
 146                                 action = -1;
 147                                 break;
 148                         case (A_REPLACE << 8) + A_RM:
 149                         case (A_MODIFY  << 8) + A_RM:
 150                                 // Prior replace/modify is irrelevant now
 151                                 break;
 152                         case (A_ADD     << 8) + A_REPLACE:
 153                         case (A_RM      << 8) + A_REPLACE:
 154                         case (A_REPLACE << 8) + A_REPLACE:
 155                         case (A_MODIFY  << 8) + A_REPLACE:
 156                                 // "Moved to" can unconditionally replace anything
 157                                 break;
 158                         case (A_ADD     << 8) + A_MODIFY:
 159                                 // Still need to make initial add
 160                         case (A_MODIFY  << 8) + A_MODIFY:
 161                                 return;
 162                         default:
 163                                 fprintf(stderr, "Not sure how to handle action %d => %d (%d %s)\n", this->action, action, flags, p);
 164                                 abort();
 165                         }
 166                         LOG_DEBUG("Change action of %s: %d =%d> %d (flags %d => %d)", p, this->action, actIn, action, this->flags, flags);
 167                         if (action == -1)
 168                         {
 169                                 *thisp = this->next;
 170                                 free(this->p);
 171                                 free(this);
 172                                 return;
 173                         }
 174                         this->action = action;
 175                         this->flags = flags;
 176                         return;
 177                 }
 178         }
 179
 180         // New entry
 181         LOG_DEBUG("Set action of %s: %d (flags %d)", p, action, flags);
 182         struct vll *n = malloc(sizeof(struct vll));
 183         n->p = strdup(p);
 184         n->action = action;
 185         n->flags = flags;
 186         n->next = *top;
 187         *top = n;
 188 }
 189
 190 static
 191 void
 192 vll_free(struct vll**top) {
 193         for (struct vll *next, *this = *top; this; this = next)
 194         {
 195                 next = this->next;
 196                 free(this->p);
 197                 free(this);
 198         }
 199         *top = NULL;
 200 }
 201
 202 struct cmdbuilder {
 203         char **cmd;
 204         char **args;
 205         size_t count;
 206         char **dcmd;
 207 };
 208
 209 // TODO: Probably a good idea to bail out early if the inotify fd is ready for reading
 210 static
 211 void
 212 vll_stage_all(struct vll**changes_p) {
 213         struct vll*changes = *changes_p;
 214         struct cmdbuilder act[1];
 215
 216         char *add[6 + MAX_ARGS] = {"git", "add", "-A", "--ignore-errors", "--"};
 217         act[A_ADD].cmd  = add;
 218         act[A_ADD].args = &add[5];
 219         act[A_ADD].count= 0;
 220
 221         #define CMD_DO(p)  \
 222         {  \
 223                 (p)->args[(p)->count] = NULL;  \
 224                 my_system2((p)->cmd[0], (p)->cmd);  \
 225         }  \
 226         // END CMD_DO
 227
 228         for (struct vll *this = changes; this; this = this->next)
 229         {
 230                 struct cmdbuilder *c;
 231                 c = &act[A_ADD];
 232
 233                 c->args[c->count++] = this->p;
 234                 if (c->count < MAX_ARGS)
 235                         continue;
 236                 CMD_DO(c)
 237                 c->count = 0;
 238         }
 239         if (act[A_ADD].count)
 240         CMD_DO(&act[A_ADD])
 241
 242         #undef CMD_DO
 243
 244         vll_free(changes_p);
 245 }
 246
 247 struct inotify_watch {
 248         char*path;
 249         int wd;
 250         struct inotify_watch*next;
 251 };
 252
 253 static
 254 void
 255 my_add_watch2(struct inotify_watch**top, int fd, const char *pathname, uint32_t mask, int recursive) {
 256         if (recursive)
 257         {
 258                 DIR *D = opendir(pathname);
 259                 assert(D || errno == ENOTDIR);
 260                 if (D)
 261                 {
 262                         errno = 0;
 263                         for (struct dirent *DE; DE = readdir(D); )
 264                         {
 265                                 if (DE->d_name[0] == '.' && ((DE->d_name[1] == '.' && DE->d_name[2] == '\0') || DE->d_name[1] == '\0'))
 266                                         continue;
 267 #ifdef IGNORE_ROOT_GIT
 268                                 if (recursive == 1 && !strcmp(".git", DE->d_name))
 269                                         continue;
 270 #endif
 271                                 if (DE->d_type == DT_DIR)
 272                                 {
 273                                         // FIXME: these snprintf/sprintf could easily be replaced with strlen etc (inline func?)
 274                                         char npn[1 + snprintf(DE->d_name, 0, "%s/%s", pathname, DE->d_name)];
 275                                         sprintf(npn, "%s/%s", pathname, DE->d_name);
 276                                         my_add_watch2(top, fd, npn, mask, 2);
 277                                 }
 278                         }
 279                         assert(!errno);
 280                         closedir(D);
 281                 }
 282         }
 283
 284         LOG_DEBUG("Adding watch on %s", pathname);
 285
 286         struct inotify_watch *n = malloc(sizeof(struct inotify_watch));
 287         n->path = strdup(pathname);
 288         n->wd = inotify_add_watch(fd, pathname, mask);
 289
 290         assert(n->wd != -1);
 291
 292         n->next = *top;
 293         *top = n;
 294 }
 295 #define my_add_watch(top, fd, path, mask) my_add_watch2(top, fd, path, mask, 0)
 296 #define my_add_watch_recursive(top, fd, path, mask) my_add_watch2(top, fd, path, mask, 1)
 297
 298 static
 299 void
 300 my_rm_watch(struct inotify_watch**top, int fd, int wd) {
 301         for (struct inotify_watch *i, **ii = top; i = *ii; ii = &i->next)
 302         {
 303                 if (i->wd == wd)
 304                 {
 305                         *ii = i->next;
 306                         inotify_rm_watch(fd, wd);
 307                         free(i->path);
 308                         free(i);
 309                         return;
 310                 }
 311         }
 312         fprintf(stderr, "Tried to remove non-existent watch: %d\n", wd);
 313         abort();
 314 }
 315
 316 static
 317 const
 318 char*
 319 get_wd_path(struct inotify_watch**top, int fd, int wd) {
 320         for (struct inotify_watch*i = *top; i; i = i->next)
 321                 if (i->wd == wd)
 322                         return i->path;
 323         return NULL;
 324 }
 325
 326 int
 327 main(int argc, char**argv) {
 328         setbuf(stdout, NULL);
 329         setbuf(stderr, NULL);
 330
 331         assert(argc > 1);
 332         const char *dir = argv[1];
 333
 334         assert(!chdir(dir));
 335         LOG_DEBUG("Changed to %s", dir);
 336
 337         int fd_inotify = inotify_init();
 338         assert(fd_inotify != -1);
 339
 340         struct inotify_watch *wlist = NULL;
 341         my_add_watch_recursive(&wlist, fd_inotify, ".", MY_WATCH_EVENTS);
 342
 343         // Initial global update check
 344         my_system("git", "add", "--all", "--ignore-errors");
 345         my_system("git", "commit", "-m", "Initial global update commit");
 346
 347         struct vll *changes = NULL;
 348
 349         int todo = 3;
 350         int commit_auto = 0;
 351
 352         char buf[BUFFER_SIZE];
 353         size_t buf_len = 0;
 354
 355         while (1)
 356         {
 357                 fd_set rfds;
 358                 FD_ZERO(&rfds);
 359                 FD_SET(fd_inotify, &rfds);
 360
 361                 struct timeval tv;
 362                 tv.tv_usec = 0;
 363                 switch (changes ? 1 : todo) {
 364                 case 0:  // nothing
 365                         LOG_DEBUG("Waiting for changes...%s","");
 366                         break;
 367                 case 1:  // add/remove
 368                         tv.tv_sec = 5;
 369                         LOG_DEBUG("Scheduling add/remove in %d seconds...", tv.tv_sec);
 370                         break;
 371                 case 2:  // commit
 372                         tv.tv_sec = 15;
 373                         LOG_DEBUG("Scheduling commit in %d seconds...", tv.tv_sec);
 374                         break;
 375                 case 3:  // push
 376                         tv.tv_sec = 30;
 377                         LOG_DEBUG("Scheduling push in %d seconds...", tv.tv_sec);
 378                         break;
 379                 }
 380
 381                 int have_inotify =
 382                 select(fd_inotify + 1, &rfds, NULL, NULL, (changes || todo) ? &tv : NULL);
 383
 384                 if (have_inotify)
 385                 {
 386                         size_t rrv = read(fd_inotify, buf, sizeof(buf));
 387                         if (rrv == -1)
 388                         {
 389                                 perror(NULL);
 390                                 exit(1);
 391                         }
 392                         buf_len = rrv;
 393                 }
 394
 395                 if (buf_len)
 396                 {
 397                         struct inotify_event *ev;
 398                         for (size_t offset = 0; offset < buf_len; offset += sizeof(struct inotify_event) + ev->len)
 399                         {
 400                                 ev = (struct inotify_event*)&buf[offset];
 401                                 // WARNING: "If successive output inotify events produced on the inotify file descriptor are identical (same wd, mask, cookie, and name) then they are coalesced into a single event."
 402
 403                                 const char *fullpath = get_wd_path(&wlist, fd_inotify, ev->wd);
 404                                 size_t fullpath_slen = 0;
 405                                 if (ev->len && ev->name[0])
 406                                         fullpath_slen = snprintf(buf, 0, "%s/%s", fullpath, ev->name) + 1;
 407                                 char fullpath_s[fullpath_slen];
 408                                 if (fullpath_slen)
 409                                 {
 410                                         sprintf(fullpath_s, "%s/%s", fullpath, ev->name);
 411                                         fullpath = fullpath_s;
 412                                 }
 413
 414 #ifdef DEBUG
 415                                 printf("Event:");
 416                                 if (ev->mask & IN_ACCESS)
 417                                         printf(" Access");
 418                                 if (ev->mask & IN_ATTRIB)
 419                                         printf(" Attrib");
 420                                 if (ev->mask & IN_CLOSE_WRITE)
 421                                         printf(" CLOSE_WRITE");
 422                                 if (ev->mask & IN_CLOSE_NOWRITE)
 423                                         printf(" CLOSE_NOWRITE");
 424                                 if (ev->mask & IN_CREATE)
 425                                         printf(" Created");
 426                                 if (ev->mask & IN_DELETE)
 427                                         printf(" Deleted");
 428                                 if (ev->mask & IN_DELETE_SELF)
 429                                         printf(" Self-deleted");
 430                                 if (ev->mask & IN_MODIFY)
 431                                         printf(" Modified");
 432                                 if (ev->mask & IN_MOVE_SELF)
 433                                         printf(" Self-move");
 434                                 if (ev->mask & IN_MOVED_FROM)
 435                                         printf(" Moved-from");
 436                                 if (ev->mask & IN_MOVED_TO)
 437                                         printf(" Moved-to");
 438                                 if (ev->mask & IN_OPEN)
 439                                         printf(" Open");
 440                                 if (ev->mask & IN_Q_OVERFLOW)
 441                                         printf(" QueueOVERFLOW");
 442                                 printf(": %s\n", fullpath);
 443 #endif
 444                                 if (ev->mask & (IN_CREATE | IN_MOVED_TO))
 445                                 {
 446                                         assert(ev->len && ev->name[0]);
 447                                         if (ev->mask & IN_ISDIR)
 448                                                 my_add_watch(&wlist, fd_inotify, fullpath, MY_WATCH_EVENTS);
 449                                                 // FIXME: only need to add watch immediately before 'git add'
 450                                         vll_set_action(&changes, fullpath, (ev->mask & IN_MOVED_TO) ? A_REPLACE : A_ADD, ev->mask);
 451                                 }
 452                                 if (ev->mask & (IN_MODIFY))
 453                                         vll_set_action(&changes, fullpath, A_MODIFY, ev->mask);
 454                                 if (ev->mask & (IN_DELETE | IN_MOVED_FROM))
 455                                         vll_set_action(&changes, fullpath, A_RM, ev->mask);
 456                                 if (ev->mask & IN_DELETE_SELF)
 457                                         my_rm_watch(&wlist, fd_inotify, ev->wd);
 458                                 if (ev->mask & IN_Q_OVERFLOW)
 459                                 {
 460                                         fprintf(stderr, "FIXME: this Q_OVERFLOW method will FAIL TO MONITOR NEW DIRECTORIES\n");
 461                                         abort();
 462                                         commit_auto = 1;
 463                                         break;
 464                                 }
 465                         }
 466 #ifdef DEBUG
 467                         puts("DONE");
 468 #endif
 469                         buf_len = 0;
 470                 }
 471
 472                 if (!have_inotify && (todo = changes ? 1 : todo))
 473                 {
 474                         if (commit_auto)
 475                         {
 476                                 my_system("git", "add", "--all");
 477                                 todo = 2;
 478                         }
 479                         switch (todo) {
 480                         case 1:
 481                                 ++todo;
 482                                 vll_stage_all(&changes);
 483                                 break;
 484                         case 2:
 485                                 ++todo;
 486                                 my_system("git", "commit", "-m", "_");
 487                                 commit_auto = 0;
 488                                 break;
 489                         case 3:
 490                                 todo = 0;
 491                                 my_system("git", "push");
 492                                 break;
 493                         }
 494                 }
 495
 496 #ifdef CHECKS
 497                 if (!changes && read_will_block(fd_inotify))
 498                 {
 499                         if (!system("git ls-files -omd | grep ."))
 500                         {
 501                                 if (read_will_block(fd_inotify))
 502                                 {
 503                                 fprintf(stderr, "git reports index de-sync!\n");
 504                                 abort();
 505                                 }
 506                         }
 507                         else
 508                                 LOG_DEBUG("git reports index fully in-sync%s","");
 509                 }
 510 #endif
 511         }
 512
 513 }