filesynced

   1 #!/bin/sh
   2
   3 #==============================================================================
   4 # filesynced
   5 # File ID: 25796c28-7205-11e5-b257-fefdb24f8e10
   6 #
   7 # Add 'synced'-entry into synced.sqlite for files.
   8 #
   9 # Author: Øyvind A. Holm <sunny@sunbase.org>
  10 # License: GNU General Public License version 2 or later.
  11 #==============================================================================
  12
  13 progname=filesynced
  14 VERSION=0.9.1
  15
  16 SQLITE=sqlite3
  17 bin="$HOME/bin"
  18 db="synced.sqlite"
  19
  20 opt_add=0
  21 opt_create_index=0
  22 opt_delete=0
  23 opt_force=0
  24 opt_help=0
  25 opt_init=0
  26 opt_list=0
  27 opt_lock=0
  28 opt_patch=0
  29 opt_set_priority=''
  30 opt_quiet=0
  31 opt_random=0
  32 opt_timeout=''
  33 opt_type='%'
  34 opt_unlock=0
  35 opt_unsynced=0
  36 opt_valid_sha=0
  37 opt_verbose=0
  38 while test -n "$1"; do
  39         case "$1" in
  40         --add) opt_add=1; shift ;;
  41         --create-index) opt_create_index=1; shift ;;
  42         --delete) opt_delete=1; shift ;;
  43         -f|--force) opt_force=1; shift ;;
  44         -h|--help) opt_help=1; shift ;;
  45         --init) opt_init=1; shift ;;
  46         -l|--list) opt_list=1; shift ;;
  47         --lock) opt_lock=1; shift ;;
  48         --patch) opt_patch=1; shift ;;
  49         -p|--set-priority) opt_set_priority=$2; shift 2 ;;
  50         -q|--quiet) opt_quiet=$(($opt_quiet + 1)); shift ;;
  51         --random) opt_random=1; shift ;;
  52         --timeout) opt_timeout=$2; shift 2 ;;
  53         -t|--type) opt_type=$2; shift 2 ;;
  54         --unlock) opt_unlock=1; shift ;;
  55         --unsynced) opt_unsynced=1; shift ;;
  56         --valid-sha) opt_valid_sha=1; shift ;;
  57         -v|--verbose) opt_verbose=$(($opt_verbose + 1)); shift ;;
  58         --version) echo $progname $VERSION; exit 0 ;;
  59         --) shift; break ;;
  60         *)
  61                 if printf '%s\n' "$1" | grep -q ^-; then
  62                         echo "$progname: $1: Unknown option" >&2
  63                         exit 1
  64                 else
  65                         break
  66                 fi
  67         break ;;
  68         esac
  69 done
  70 opt_verbose=$(($opt_verbose - $opt_quiet))
  71
  72 if test "$opt_help" = "1"; then
  73         test $opt_verbose -gt 0 && { echo; echo $progname $VERSION; }
  74         cat <<END
  75
  76 Add 'synced'-entry into $db for files. Updates synced.rev and
  77 synced.date with Git commit info.
  78
  79 Usage: $progname [OPTIONS] COMMIT FILE [FILE [...]]
  80        $progname --init
  81        $progname --add [-t TYPE] FILE [FILE [...]]
  82        $progname --create-index
  83        $progname --delete FILE [FILE [...]]
  84        $progname -l [-t TYPE]
  85        $progname -p NEWPRI FILE [FILES [...]]
  86        $progname --lock
  87        $progname --unlock TOKEN
  88        $progname --unlock -f
  89        $progname --unsynced [OPTIONS] [FILES] [ -- GIT_OPTIONS ]
  90        $progname --valid-sha
  91
  92 Options:
  93
  94   --add
  95     Add all files specified on the command line to synced.sql, use
  96     undefined sync values.
  97   --create-index
  98     Create database indexes to speed up operations when synced.sql is
  99     getting quite big.
 100   --delete
 101     Delete all files specified on the commandline from the database.
 102   -f, --force
 103     When used together with --unlock, force unlock without a token.
 104   -h, --help
 105     Show this help.
 106   --init
 107     Initialise the current Git repo for use with $progname, create
 108     synced.sql in the top directory of the repository.
 109   -l, --list
 110     Create a file list sorted by how many revisions they are behind the
 111     files in Lib/std/ in current HEAD.
 112   --lock
 113     Create $db from synced.sql and activate locking. A lock
 114     token is sent to stdout, and this token is needed to unlock.
 115   --patch
 116     Try to apply the missing patches directly to the files. Files that
 117     were successfully patched are added to Git, together with an updated
 118     entry in synced.sql. Files with conflicts are not added, so it's
 119     easy to find conflicts with a standard "git diff".
 120   -p NEWPRI, --set-priority NEWPRI
 121     Change todo priority for files specified on the command line. NEWPRI
 122     must be between 1 and 5.
 123   -q, --quiet
 124     Be more quiet. Can be repeated to increase silence.
 125   --random
 126     Pick a random file from the todo list and sync it using "vd".
 127   --timeout SECONDS
 128     Wait maximum SECONDS seconds for lock. Default is no timeout, wait
 129     until it gets the lock or the script is forcibly terminated.
 130   -t FILETYPE. --type FILETYPE
 131     Limit list to files of type FILETYPE, for example "bash" or
 132     "perl-tests". SQL LIKE wildcards can be used, like '%' and '_'.
 133     If used together with --add, set the 'orig' field in the database to
 134     this value, prefixed with 'Lib/std/'.
 135   --unlock TOKEN
 136   --unlock -f
 137     If TOKEN is correct, overwrite synced.sql with the contents from
 138     $db, delete synced.slite and remove lock. To unlock without a token,
 139     add the -f/--force option.
 140   --unsynced
 141     Generate a list of all filenames that don't have 'rev' defined in
 142     synced.sql, i.e., those that haven't been synced yet. Arguments are
 143     delivered to git-allfiles(1). --since is useful with this, for
 144     example "--since=1.year". Options meant for git-allfiles must come
 145     after a " -- " to separate them from the argument parsing done by
 146     $progname.
 147   --valid-sha
 148     Test that all 'rev' SHA1s in synced.sql are reachable from the
 149     current Git HEAD. For example, execute this from 'master' to make
 150     sure that no files are synced against revisions on branches that are
 151     not part of it.
 152   -v, --verbose
 153     Increase level of verbosity. Can be repeated.
 154   --version
 155     Print version information.
 156
 157 END
 158         exit 0
 159 fi
 160
 161 msg() {
 162         local empty_line=
 163         local no_lf=
 164         local prefix_str=
 165         if test $opt_verbose -gt 2; then
 166                 prefix_str="$progname $$:$BASH_LINENO: "
 167         else
 168                 prefix_str="$progname: "
 169         fi
 170         if test "$1" = "-l"; then
 171                 # If -l is specified, create empty line before message
 172                 empty_line=1
 173                 shift
 174         fi
 175         if test "$1" = "-n"; then
 176                 # If -n is specified, don't terminate with \n
 177                 no_lf="-n"
 178                 shift
 179         fi
 180         if test "$1" = "-q"; then
 181                 # -q suppresses $progname prefix
 182                 prefix_str=""
 183                 shift
 184         fi
 185         local vlevel=$1
 186         shift
 187         test $vlevel -gt $opt_verbose && return;
 188         test "$empty_line" = "1" && echo >&2
 189         echo $no_lf "$prefix_str$*" >&2
 190         return
 191 }
 192
 193 msg -l 2 Starting $progname, pwd = $(pwd)
 194
 195 if test -n "$opt_timeout"; then
 196         if test -n "$(echo "$opt_timeout" | tr -d 0-9 | grep .)"; then
 197                 echo $progname: Argument to --timeout must be an integer >&2
 198                 exit 1
 199         fi
 200 fi
 201
 202 repotop="$(git rev-parse --show-toplevel)"
 203 msg 2 repotop = \"$repotop\"
 204 lockdir="$repotop/synced.sql.lock"
 205 msg 2 lockdir = \"$lockdir\"
 206
 207 cleanup() {
 208         msg 2 "cleanup(): bintoken = $bintoken"
 209         if test -n "$bintoken"; then
 210                 msg 2 Unlock bintoken
 211                 cd "$bin" && filesynced --unlock $bintoken
 212         fi
 213         msg 2 Unlock token
 214         cd "$repotop" && filesynced --unlock $token
 215 }
 216
 217 init_db() {
 218         cat <<END | $SQLITE "$1"
 219 CREATE TABLE synced (
 220   file TEXT
 221     CONSTRAINT synced_file_length
 222       CHECK (length(file) > 0)
 223     UNIQUE
 224     NOT NULL
 225   ,
 226   orig TEXT
 227   ,
 228   rev TEXT
 229     CONSTRAINT synced_rev_length
 230       CHECK (length(rev) = 40 OR rev = '')
 231   ,
 232   date TEXT
 233     CONSTRAINT synced_date_length
 234       CHECK (date IS NULL OR length(date) = 19)
 235     CONSTRAINT synced_date_valid
 236       CHECK (date IS NULL OR datetime(date) IS NOT NULL)
 237 );
 238 CREATE TABLE todo (
 239   file TEXT
 240     CONSTRAINT todo_file_length
 241       CHECK(length(file) > 0)
 242     UNIQUE
 243     NOT NULL
 244   ,
 245   pri INTEGER
 246     CONSTRAINT todo_pri_range
 247       CHECK(pri BETWEEN 1 AND 5)
 248   ,
 249   comment TEXT
 250 );
 251 END
 252 }
 253
 254 safe_chdir() {
 255         local dir="$1"
 256         msg 2 chdir $dir
 257         cd "$dir" || {
 258                 echo $progname: Cannot chdir to \'$dir\' >&2
 259                 exit 1
 260         }
 261 }
 262
 263 if test "$opt_lock" = "1" -a "$opt_unlock" = "1"; then
 264         echo $progname: Cannot mix --lock and --unlock >&2
 265         exit 1
 266 fi
 267
 268 if test "$opt_lock" = "1"; then
 269         msg 2 Perform --lock stuff
 270         cd "$repotop" || {
 271                 echo $progname --lock: $repotop: chdir error >&2
 272                 exit 1
 273         }
 274         begin_wait_time=$(date -u +%s)
 275         until mkdir "$lockdir" 2>/dev/null; do
 276                 echo $progname --lock: $lockdir: Waiting for lockdir... >&2
 277                 if test -n "$opt_timeout" -a \
 278                   $(( $(date -u +%s) - $begin_wait_time )) -gt \
 279                   $(( $opt_timeout - 1 )); then
 280                         printf '%s: Lock not aquired after %u second%s, aborting\n' >&2 \
 281                             "$progname" \
 282                             "$opt_timeout" \
 283                             "$(test "$opt_timeout" = "1" || echo -n s)"
 284                         exit 1
 285                 fi
 286                 sleep 2
 287         done
 288         if test -e "$db"; then
 289                 echo $progname --lock: $repotop/$db: File already exists >&2
 290                 rmdir "$lockdir"
 291                 exit 1
 292         fi
 293         if test -f "synced.sql"; then
 294                 msg 2 Create $db from $(pwd)/synced.sql
 295                 $SQLITE "$db" <synced.sql
 296         fi
 297         token="token_$(date -u +"%Y%m%dT%H%M%SZ").$$"
 298         echo $token >"$lockdir/token"
 299         echo $token
 300         exit 0
 301 fi
 302
 303 if test "$opt_unlock" = "1"; then
 304         msg 2 Perform --unlock stuff
 305         token_from_user="$1"
 306         msg 2 token_from_user = $token_from_user
 307         cd "$repotop" || {
 308                 echo $progname --unlock: $repotop: chdir error >&2
 309                 exit 1
 310         }
 311         if test ! -d "$lockdir"; then
 312                 echo $progname --unlock: $lockdir: Lockdir doesn\'t exist >&2
 313                 exit 1
 314         fi
 315         if test -e "$lockdir/token"; then
 316                 msg 2 $lockdir/token exists
 317                 realtoken=$(cat $lockdir/token)
 318                 msg 2 realtoken = $realtoken
 319                 if test "$token_from_user" != "$realtoken" -a \
 320                   "$opt_force" != "1" ; then
 321                         echo $progname --unlock: Token mismatch >&2
 322                         msg 2 Got $token_from_user
 323                         msg 2 Expected $realtoken
 324                         exit 1
 325                 fi
 326                 msg 2 Token is valid, delete $lockdir/token
 327                 rm "$lockdir/token"
 328         fi
 329         if test -f "$db"; then
 330                 echo "
 331 BEGIN EXCLUSIVE TRANSACTION;
 332
 333 CREATE TEMPORARY TABLE tmp AS
 334   SELECT * FROM synced;
 335 DELETE FROM synced;
 336 INSERT INTO synced
 337   SELECT * FROM tmp ORDER BY file;
 338 DROP TABLE tmp;
 339
 340 CREATE TEMPORARY TABLE tmp AS
 341   SELECT * FROM todo;
 342 DELETE FROM todo;
 343 INSERT INTO todo
 344   SELECT * FROM tmp ORDER BY file;
 345 DROP TABLE tmp;
 346
 347 COMMIT TRANSACTION;
 348                 " | $SQLITE "$db" || {
 349                         echo $progname: SQLite error, cannot sort tables >&2
 350                         exit 1
 351                 }
 352                 msg 2 Dump $db to $(pwd)/synced.sql
 353                 $SQLITE "$db" .dump >synced.sql
 354                 msg 2 Remove $db
 355                 rm "$db"
 356         else
 357                 msg 1 $db not found, did not update synced.sql
 358         fi
 359         if rmdir "$lockdir"; then
 360                 msg 2 $lockdir removed
 361                 exit 0
 362         else
 363                 echo $progname --unlock: $lockdir: Could not remove lockdir >&2
 364                 exit 1
 365         fi
 366 fi
 367
 368 if test "$opt_unsynced" = "1"; then
 369         msg 2 Perform --unsynced stuff
 370         token=$(filesynced --lock)
 371         if test -z "$(echo $token | grep ^token_)"; then
 372                 echo $progname --unsynced: Cannot --lock >&2
 373                 exit 1
 374         fi
 375         git allfiles "$@" | strip-nonexisting | while read f; do
 376                 cat <<END
 377 SELECT file FROM synced
 378   WHERE
 379     file='$f'
 380     AND
 381     rev IS NULL;
 382 END
 383         done | sqlite3 synced.sqlite
 384         filesynced --unlock $token
 385         exit
 386 fi
 387
 388 if test "$opt_create_index" = "1"; then
 389         msg 2 Perform --create-index stuff
 390         token=$(filesynced --lock)
 391         if test -z "$(echo $token | grep ^token_)"; then
 392                 echo $progname --create-index: Cannot --lock >&2
 393                 exit 1
 394         fi
 395         cat <<END | $SQLITE "$db"
 396 CREATE INDEX IF NOT EXISTS idx_synced_file ON synced (file);
 397 CREATE INDEX IF NOT EXISTS idx_synced_orig ON synced (orig);
 398 CREATE INDEX IF NOT EXISTS idx_synced_rev ON synced (rev);
 399 END
 400         filesynced --unlock $token
 401         exit
 402 fi
 403
 404 if test "$opt_patch" = "1"; then
 405         msg 2 Perform --patch stuff
 406         # FIXME: Make it work with filenames containing bloody whitespace.
 407         # Will take care of that when the output format of -l/--list changes
 408         # after --patch is in place.
 409         files_behind="$(
 410             filesynced -l |
 411             grep -v ^0 |
 412             awk '{ print $2 }'
 413         )"
 414         if test -z "$files_behind"; then
 415                 msg 0 No files need patching
 416                 exit 0
 417         fi
 418         echo "$files_behind" | while read f; do
 419                 echo
 420                 token=$(filesynced --lock)
 421                 if test -z "$(echo $token | grep ^token_)"; then
 422                         echo $progname --patch: Cannot --lock >&2
 423                         exit 1
 424                 fi
 425                 orig=$($SQLITE "$db" "SELECT orig FROM synced WHERE file = '$f';")
 426                 msg 2 orig = \"$orig\"
 427                 rev=$($SQLITE "$db" "SELECT rev FROM synced WHERE file = '$f';")
 428                 msg 2 rev = \"$rev\"
 429                 filesynced --unlock $token
 430
 431                 (cd ~/bin && git diff $rev.. $orig) |
 432                   patch -m --no-backup-if-mismatch "$f" && (
 433                     filesynced HEAD "$f"
 434                     git add "$f" synced.sql
 435                 )
 436         done
 437         exit
 438 fi
 439
 440 bintoken=
 441 token=$(filesynced --lock --timeout "$opt_timeout")
 442 if test -z "$(echo $token | grep ^token_)"; then
 443         echo $progname: No token received from filesynced --lock >&2
 444         exit 1
 445 fi
 446 trap cleanup EXIT
 447 msg 2 token = $token
 448
 449 if test "$opt_init" = "1"; then
 450         msg 2 Perform --init stuff
 451         safe_chdir "$repotop"
 452         for f in synced.sql synced.sqlite; do
 453                 if test -e "$f"; then
 454                         echo $progname --init: $repotop/$f already exists >&2
 455                         exit 1
 456                 fi
 457         done
 458         init_db synced.sqlite
 459         $SQLITE synced.sqlite .dump >synced.sql
 460         rm synced.sqlite
 461         exit
 462 fi
 463
 464 test -f "$db" || {
 465         echo $progname: $db: Sync database not found >&2
 466         exit 1
 467 }
 468
 469 if test "$opt_list" = "1"; then
 470         msg 2 Perform --list stuff
 471         test -n "$opt_type" && type_str="$opt_type" || type_str="%"
 472         safe_chdir "$bin"
 473         if test "$(cat synced.sql.lock/token 2>/dev/null)" != "$token"; then
 474                 msg 2 token is different from synced.sql.lock/token
 475                 bintoken=$(filesynced --lock)
 476                 msg 2 bintoken = $bintoken
 477                 if test -z "$(echo $bintoken | grep ^token_)"; then
 478                         echo -n "$progname: No token received " >&2
 479                         echo from filesynced --lock in $bin >&2
 480                         exit 1
 481                 fi
 482         fi
 483         safe_chdir - >/dev/null
 484         cat <<END | $SQLITE "$db" | bash | sort -n
 485 SELECT
 486   'echo \$(' ||
 487     'cd "$bin"; git log --format=%h ' || rev || '.. ' || orig || ' | wc -l' ||
 488   ') ' ||
 489   file ||
 490   ' "(' ||
 491     'cd ~/bin && git diff ' ||
 492     '\$(cd "$bin"; git log -1 --format=%h ' || rev || ')' ||
 493     '.. ' ||
 494     orig ||
 495   ') | patch -m ' || file || ' && filesynced HEAD ' || file || '";'
 496 FROM synced
 497 WHERE
 498   orig LIKE 'Lib/std/$type_str'
 499   AND
 500   rev IS NOT NULL;
 501 END
 502         exit
 503 fi
 504
 505 if test "$opt_delete" = "1"; then
 506         msg 2 Perform --delete stuff
 507         safe_chdir "$repotop"
 508         retval=0
 509         for f in "$@"; do
 510                 from_synced="$(
 511                     echo "SELECT file FROM synced WHERE file = '$f';" | $SQLITE "$db"
 512                 )"
 513                 from_todo="$(
 514                     echo "SELECT file FROM todo WHERE file = '$f';" | $SQLITE "$db"
 515                 )"
 516                 if test -n "$from_synced"; then
 517                         echo "DELETE FROM synced WHERE file = '$f';" | $SQLITE "$db"
 518                         msg 0 Deleted $f from synced
 519                 else
 520                         retval=1
 521                 fi
 522                 if test -n "$from_todo"; then
 523                         echo "DELETE FROM todo WHERE file = '$f';" | $SQLITE "$db"
 524                         msg 0 Deleted $f from todo
 525                 fi
 526         done
 527         exit $retval
 528 fi
 529
 530 if test "$opt_random" = "1"; then
 531         msg 2 Perform --random stuff
 532         file="$(
 533                 $SQLITE "$db" "
 534 SELECT file FROM todo
 535   ORDER BY pri, random()
 536   LIMIT 1;
 537                 "
 538         )"
 539         if test -z "$file"; then
 540                 echo $progname: No files to edit >&2
 541                 exit 0
 542         fi
 543         vd "$file" "$HOME/bin/$(
 544             $SQLITE $db "
 545 SELECT orig FROM synced
 546   WHERE file = '$file';
 547             "
 548         )"
 549         exit
 550 fi
 551
 552 if test -n "$opt_set_priority"; then
 553         msg 2 Perform --set-priority stuff
 554         echo "$opt_set_priority" | grep -qE '^[1-5]$' || {
 555                 echo $progname: Argument to -p/--set-priority must be between 1 and 5
 556                 exit 1
 557         }
 558         (
 559                 echo BEGIN\;
 560                 for f in "$@"; do
 561                         echo "UPDATE todo SET pri = $opt_set_priority WHERE file = '$f';"
 562                 done
 563                 echo END\;
 564         ) | $SQLITE "$db"
 565         exit
 566 fi
 567
 568 if test "$opt_valid_sha" = "1"; then
 569         msg 2 Perform --valid-sha stuff
 570         cat <<END | sqlite3 synced.sqlite | sort -u | grep . | while read f; do
 571 SELECT rev FROM synced
 572   WHERE
 573     rev IS NOT NULL;
 574 END
 575                 if test "$(git log --format=%h ..$f | wc -l)" != "0"; then
 576                         echo $f
 577                 fi
 578         done
 579         exit
 580 fi
 581
 582 ref=$1
 583
 584 if test "$opt_add" != "1"; then
 585         commit="$(cd "$bin"; git rev-parse $1)"
 586         if test -z "$commit"; then
 587                 echo $progname: $1: Invalid Git ref >&2
 588                 exit 1
 589         fi
 590         shift
 591 fi
 592 files="$*"
 593 if test -z "$files"; then
 594         echo $progname: No files specified >&2
 595         exit 1
 596 fi
 597 date="$(
 598         cd "$bin"
 599         git log -1 --format=%cd --date=raw $commit |
 600         cut -f 1 -d ' ' |
 601         ep -D ' ' |
 602         tr -d Z
 603 )"
 604
 605 for f in $files; do
 606         if test "$opt_add" = "1"; then
 607                 if test ! -f "$f"; then
 608                         echo $progname: $f: File not found, no entries updated >&2
 609                         exit 1
 610                 fi
 611         else
 612                 if test -z "$(git ls-files "$f")"; then
 613                         echo $progname: $f: File is not in Git, no entries updated >&2
 614                         exit 1
 615                 fi
 616         fi
 617 done
 618
 619 (
 620         echo BEGIN\;
 621         for f in $files; do
 622                 if test "$opt_add" = "1"; then
 623                         if test "$opt_type" != "%"; then
 624                                 type_def=",orig"
 625                                 type_val=",'Lib/std/$opt_type'"
 626                         else
 627                                 unset type_def type_val
 628                         fi
 629                         echo "INSERT INTO synced (file$type_def) VALUES ('$f'$type_val);" |
 630                           $SQLITE "$db"
 631                         if test "$?" != "0"; then
 632                                 echo -n "$progname: Cannot add \"$f\" to the database, " >&2
 633                                 echo no entries updated >&2
 634                                 echo "ROLLBACK;"
 635                         fi
 636                 else
 637                         if test "$ref" = "HEAD"; then
 638                                 sql_delete_todo="DELETE FROM todo WHERE file = '$f';"
 639                         else
 640                                 sql_delete_todo=
 641                         fi
 642                         cat <<END
 643 UPDATE synced
 644   SET rev = '$commit', date = '$date'
 645   WHERE file = '$f';
 646 $sql_delete_todo
 647 END
 648                 fi
 649         done
 650         echo COMMIT\;
 651 ) | $SQLITE "$db"
 652
 653 # vim: set ts=8 sw=8 sts=8 noet fo+=w tw=79 fenc=UTF-8 :