scripts/git/git-setup-dirs.sh

   1 #!/usr/bin/env bash
   2
   3 SCRIPT_NAME=$(basename "$0")
   4
   5 function usage()
   6 {
   7   echo "$SCRIPT_NAME [-h] [-n] [-u]"
   8   echo
   9   echo "  arguments:"
  10   echo "   -h: show this help text"
  11   echo "   -n: dry run mode"
  12   echo "       (default: run commands)"
  13   echo "   -u: if a directory or worktree already exists, use it"
  14   echo "       (default: fail and exit on existing directories)"
  15   echo
  16   echo " env vars:"
  17   echo "   required:"
  18   echo "   TOR_FULL_GIT_PATH: where the git repository directories reside."
  19   echo "       You must set this env var, we recommend \$HOME/git/"
  20   echo "       (default: fail if this env var is not set;"
  21   echo "       current: $GIT_PATH)"
  22   echo
  23   echo "   optional:"
  24   echo "   TOR_MASTER: the name of the directory containing the tor.git clone"
  25   echo "       The primary tor git directory is \$GIT_PATH/\$TOR_MASTER"
  26   echo "       (default: tor; current: $TOR_MASTER_NAME)"
  27   echo "   TOR_WKT_NAME: the name of the directory containing the tor"
  28   echo "       worktrees. The tor worktrees are:"
  29   echo "       \$GIT_PATH/\$TOR_WKT_NAME/{maint-*,release-*}"
  30   echo "       (default: tor-wkt; current: $TOR_WKT_NAME)"
  31   echo "   TOR_GIT_ORIGIN_PULL: the origin remote pull URL."
  32   echo "       (current: $GIT_ORIGIN_PULL)"
  33   echo "   TOR_GIT_ORIGIN_PUSH: the origin remote push URL"
  34   echo "       (current: $GIT_ORIGIN_PUSH)"
  35   echo "   TOR_UPSTREAM_REMOTE_NAME: the default upstream remote."
  36   echo "       If \$TOR_UPSTREAM_REMOTE_NAME is not 'origin', we have a"
  37   echo "       separate upstream remote, and we don't push to origin."
  38   echo "       (default: $DEFAULT_UPSTREAM_REMOTE)"
  39   echo "   TOR_GITHUB_PULL: the tor-github remote pull URL"
  40   echo "       (current: $GITHUB_PULL)"
  41   echo "   TOR_GITHUB_PUSH: the tor-github remote push URL"
  42   echo "       (current: $GITHUB_PUSH)"
  43   echo "   TOR_EXTRA_CLONE_ARGS: extra arguments to git clone"
  44   echo "       (current: $TOR_EXTRA_CLONE_ARGS)"
  45   echo "   TOR_EXTRA_REMOTE_NAME: the name of an extra remote"
  46   echo "       This remote is not pulled by this script or git-pull-all.sh."
  47   echo "       This remote is not pushed by git-push-all.sh."
  48   echo "       (current: $TOR_EXTRA_REMOTE_NAME)"
  49   echo "   TOR_EXTRA_REMOTE_PULL: the extra remote pull URL."
  50   echo "       (current: $TOR_EXTRA_REMOTE_PULL)"
  51   echo "   TOR_EXTRA_REMOTE_PUSH: the extra remote push URL"
  52   echo "       (current: $TOR_EXTRA_REMOTE_PUSH)"
  53   echo "   we recommend that you set these env vars in your ~/.profile"
  54 }
  55
  56 #################
  57 # Configuration #
  58 #################
  59
  60 # Don't change this configuration - set the env vars in your .profile
  61
  62 # Where are all those git repositories?
  63 GIT_PATH=${TOR_FULL_GIT_PATH:-"FULL_PATH_TO_GIT_REPOSITORY_DIRECTORY"}
  64 # The primary tor git repository directory from which all the worktree have
  65 # been created.
  66 TOR_MASTER_NAME=${TOR_MASTER_NAME:-"tor"}
  67 # The worktrees location (directory).
  68 TOR_WKT_NAME=${TOR_WKT_NAME:-"tor-wkt"}
  69
  70 # Origin repositories
  71 GIT_ORIGIN_PULL=${TOR_GIT_ORIGIN_PULL:-"https://gitlab.torproject.org/tpo/core/tor.git"}
  72 GIT_ORIGIN_PUSH=${TOR_GIT_ORIGIN_PUSH:-"git@git-rw.torproject.org:tor.git"}
  73 # The upstream remote which gitlab.torproject.org/tpo/core/tor.git points to.
  74 DEFAULT_UPSTREAM_REMOTE=${TOR_UPSTREAM_REMOTE_NAME:-"upstream"}
  75 # Copy the URLs from origin
  76 GIT_UPSTREAM_PULL="$GIT_ORIGIN_PULL"
  77 GIT_UPSTREAM_PUSH="$GIT_ORIGIN_PUSH"
  78 # And avoid pushing to origin if we have an upstream
  79 if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then
  80   GIT_ORIGIN_PUSH="No pushes to origin, if there is an upstream"
  81 fi
  82 # GitHub repositories
  83 GITHUB_PULL=${TOR_GITHUB_PULL:-"https://github.com/torproject/tor.git"}
  84 GITHUB_PUSH=${TOR_GITHUB_PUSH:-"No_Pushing_To_GitHub"}
  85
  86 ##########################
  87 # Git branches to manage #
  88 ##########################
  89
  90 # The branches and worktrees need to be modified when there is a new branch,
  91 # and when an old branch is no longer supported.
  92
  93 set -e
  94 eval "$(git-list-tor-branches.sh -b)"
  95 set +e
  96
  97 # The main branch path has to be the main repository thus contains the
  98 # origin that will be used to fetch the updates. All the worktrees are created
  99 # from that repository.
 100 ORIGIN_PATH="$GIT_PATH/$TOR_MASTER_NAME"
 101
 102 #######################
 103 # Argument processing #
 104 #######################
 105
 106 # Controlled by the -n option. The dry run option will just output the command
 107 # that would have been executed for each worktree.
 108 DRY_RUN=0
 109
 110 # Controlled by the -s option. The use existing option checks for existing
 111 # directories, and re-uses them, rather than creating a new directory.
 112 USE_EXISTING=0
 113 USE_EXISTING_HINT="Use existing: '$SCRIPT_NAME -u'."
 114
 115 while getopts "hnu" opt; do
 116   case "$opt" in
 117     h) usage
 118        exit 0
 119        ;;
 120     n) DRY_RUN=1
 121        echo "    *** DRY RUN MODE ***"
 122        ;;
 123     u) USE_EXISTING=1
 124        echo "    *** USE EXISTING DIRECTORIES MODE ***"
 125        ;;
 126     *)
 127        echo
 128        usage
 129        exit 1
 130        ;;
 131   esac
 132 done
 133
 134 ###########################
 135 # Git worktrees to manage #
 136 ###########################
 137
 138 COUNT=${#WORKTREE[@]}
 139
 140 #############
 141 # Constants #
 142 #############
 143
 144 # Control characters
 145 CNRM=$'\x1b[0;0m'   # Clear color
 146
 147 # Bright color
 148 BGRN=$'\x1b[1;32m'
 149 BBLU=$'\x1b[1;34m'
 150 BRED=$'\x1b[1;31m'
 151 BYEL=$'\x1b[1;33m'
 152 IWTH=$'\x1b[3;37m'
 153
 154 # Strings for the pretty print.
 155 MARKER="${BBLU}[${BGRN}+${BBLU}]${CNRM}"
 156 SUCCESS="${BGRN}success${CNRM}"
 157 SKIPPED="${BYEL}skipped${CNRM}"
 158 FAILED="${BRED}failed${CNRM}"
 159
 160 ####################
 161 # Helper functions #
 162 ####################
 163
 164 # Validate the given returned value (error code), print success or failed. The
 165 # second argument is the error output in case of failure, it is printed out.
 166 # On failure, this function exits.
 167 function validate_ret
 168 {
 169   if [ "$1" -eq 0 ]; then
 170     printf "%s\\n" "$SUCCESS"
 171   else
 172     printf "%s\\n" "$FAILED"
 173     printf "      %s\\n" "$2"
 174     exit 1
 175   fi
 176 }
 177
 178 # Validate the given returned value (error code), print success, skipped, or
 179 # failed. If $USE_EXISTING is 0, fail on error, otherwise, skip on error.
 180 # The second argument is the error output in case of failure, it is printed
 181 # out. On failure, this function exits.
 182 function validate_ret_skip
 183 {
 184   if [ "$1" -ne 0 ]; then
 185     if [ "$USE_EXISTING" -eq "0" ]; then
 186       # Fail and exit with error
 187       validate_ret "$1" "$2 $USE_EXISTING_HINT"
 188     else
 189       printf "%s\\n" "$SKIPPED"
 190       printf "      %s\\n" "${IWTH}$2${CNRM}"
 191       # Tell the caller to skip the rest of the function
 192       return 0
 193     fi
 194   fi
 195   # Tell the caller to continue
 196   return 1
 197 }
 198
 199 # Create a directory, and any missing enclosing directories.
 200 # If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
 201 function make_directory
 202 {
 203   local cmd="mkdir -p '$1'"
 204   printf "  %s Creating directory %s..." "$MARKER" "$1"
 205   local check_cmd="[ ! -d '$1' ]"
 206   msg=$( eval "$check_cmd" 2>&1 )
 207   if validate_ret_skip $? "Directory already exists."; then
 208     return
 209   fi
 210   if [ $DRY_RUN -eq 0 ]; then
 211     msg=$( eval "$cmd" 2>&1 )
 212     validate_ret $? "$msg"
 213   else
 214     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 215   fi
 216 }
 217
 218 # Create a symlink from the first argument to the second argument
 219 # If the link already exists: fail if $USE_EXISTING is 0, otherwise skip.
 220 function make_symlink
 221 {
 222   local cmd="ln -s '$1' '$2'"
 223   printf "  %s Creating symlink from %s to %s..." "$MARKER" "$1" "$2"
 224   local check_cmd="[ ! -e '$2' ]"
 225   msg=$( eval "$check_cmd" 2>&1 )
 226   if validate_ret_skip $? "File already exists."; then
 227     return
 228   fi
 229   if [ $DRY_RUN -eq 0 ]; then
 230     msg=$( eval "$cmd" 2>&1 )
 231     validate_ret $? "$msg"
 232   else
 233     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 234   fi
 235 }
 236
 237 # Go into the directory or repository, even if $DRY_RUN is non-zero.
 238 # If the directory does not exist, fail and log an error.
 239 # Otherwise, silently succeed.
 240 function goto_dir
 241 {
 242   if ! cd "$1" 1>/dev/null 2>/dev/null ; then
 243     printf "  %s Changing to directory %s..." "$MARKER" "$1"
 244     validate_ret 1 "$1: Not found. Stopping."
 245   fi
 246 }
 247
 248 # Clone a repository into a directory.
 249 # If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
 250 function clone_repo
 251 {
 252   local cmd="git clone $TOR_EXTRA_CLONE_ARGS '$1' '$2'"
 253   printf "  %s Cloning %s into %s..." "$MARKER" "$1" "$2"
 254   local check_cmd="[ ! -d '$2' ]"
 255   msg=$( eval "$check_cmd" 2>&1 )
 256   if validate_ret_skip $? "Directory already exists."; then
 257     # If we skip the clone, we need to do a fetch
 258     goto_dir "$ORIGIN_PATH"
 259     fetch_remote "origin"
 260     return
 261   fi
 262   if [ $DRY_RUN -eq 0 ]; then
 263     msg=$( eval "$cmd" 2>&1 )
 264     validate_ret $? "$msg"
 265   else
 266     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 267   fi
 268 }
 269
 270 # Add a remote by name and URL.
 271 # If the remote already exists: fail if $USE_EXISTING is 0, otherwise skip.
 272 function add_remote
 273 {
 274   local cmd="git remote add '$1' '$2'"
 275   printf "  %s Adding remote %s at %s..." "$MARKER" "$1" "$2"
 276   local check_cmd="git remote get-url '$1'"
 277   msg=$( eval "$check_cmd" 2>&1 )
 278   ret=$?
 279   # We don't want a remote, so we invert the exit status
 280   if validate_ret_skip $(( ! ret )) \
 281                        "Remote already exists for $1 at $msg."; then
 282     return
 283   fi
 284   if [ $DRY_RUN -eq 0 ]; then
 285     msg=$( eval "$cmd" 2>&1 )
 286     validate_ret $? "$msg"
 287   else
 288     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 289   fi
 290 }
 291
 292 # Set a remote's push URL by name and URL.
 293 function set_remote_push
 294 {
 295   local cmd="git remote set-url --push '$1' '$2'"
 296   printf "  %s Setting remote %s push URL to '%s'..." "$MARKER" "$1" "$2"
 297   if [ $DRY_RUN -eq 0 ]; then
 298     msg=$( eval "$cmd" 2>&1 )
 299     validate_ret $? "$msg"
 300   else
 301     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 302   fi
 303 }
 304
 305 # Fetch a remote by name.
 306 function fetch_remote
 307 {
 308   local cmd="git fetch '$1'"
 309   printf "  %s Fetching %s..." "$MARKER" "$1"
 310   if [ $DRY_RUN -eq 0 ]; then
 311     msg=$( eval "$cmd" 2>&1 )
 312     validate_ret $? "$msg"
 313   else
 314     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 315   fi
 316 }
 317
 318 # Replace the fetch configs for a remote with config if they match a pattern.
 319 function replace_fetch_config
 320 {
 321   local cmd="git config --replace-all remote.'$1'.fetch '$2' '$3'"
 322   printf "  %s Replacing %s fetch configs for '%s'..." \
 323     "$MARKER" "$1" "$3"
 324   if [ $DRY_RUN -eq 0 ]; then
 325     msg=$( eval "$cmd" 2>&1 )
 326     validate_ret $? "$msg"
 327   else
 328     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 329   fi
 330 }
 331
 332 # Set up the tor-github PR config, so tor-github/pr/NNNN/head points to GitHub
 333 # PR NNNN. In some repositories, "/head" is optional.
 334 function set_tor_github_pr_fetch_config
 335 {
 336   # Standard branches
 337   replace_fetch_config tor-github \
 338     "+refs/heads/*:refs/remotes/tor-github/*" \
 339     "refs/heads"
 340   # PRs
 341   replace_fetch_config "tor-github" \
 342     "+refs/pull/*:refs/remotes/tor-github/pr/*" \
 343     "refs/pull.*pr"
 344 }
 345
 346 # Set up the tor-github PR config, so tor-gitlab/mr/NNNN points to GitHub
 347 # MR NNNN. In some repositories, "/head" is optional.
 348 function set_tor_gitlab_mr_fetch_config
 349 {
 350   # standard branches
 351   replace_fetch_config tor-gitlab \
 352     "+refs/heads/*:refs/remotes/tor-gitlab/*" \
 353     "refs/heads"
 354   # MRs
 355   replace_fetch_config tor-gitlab \
 356     "+refs/merge-requests/*/head:refs/remotes/tor-gitlab/mr/*" \
 357     "refs/merge-requests.*mr"
 358 }
 359
 360 # Add a new worktree for branch at path.
 361 # If the directory already exists: fail if $USE_EXISTING is 0, otherwise skip.
 362 function add_worktree
 363 {
 364   local cmd="git worktree add '$2' '$1'"
 365   printf "  %s Adding worktree for %s at %s..." "$MARKER" "$1" "$2"
 366   local check_cmd="[ ! -d '$2' ]"
 367   msg=$( eval "$check_cmd" 2>&1 )
 368   if validate_ret_skip $? "Directory already exists."; then
 369     return
 370   fi
 371   if [ $DRY_RUN -eq 0 ]; then
 372     msg=$( eval "$cmd" 2>&1 )
 373     validate_ret $? "$msg"
 374   else
 375     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 376   fi
 377 }
 378
 379 # Switch to the given branch name.
 380 # If the branch does not exist: fail.
 381 function switch_branch
 382 {
 383   local cmd="git checkout '$1'"
 384   printf "  %s Switching branch to %s..." "$MARKER" "$1"
 385   if [ $DRY_RUN -eq 0 ]; then
 386     msg=$( eval "$cmd" 2>&1 )
 387     validate_ret $? "$msg"
 388   else
 389     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 390   fi
 391 }
 392
 393 # Checkout a new branch with the given branch name.
 394 # If the branch already exists: fail if $USE_EXISTING is 0, otherwise skip.
 395 function new_branch
 396 {
 397   local cmd="git checkout -b '$1'"
 398   printf "  %s Creating new branch %s..." "$MARKER" "$1"
 399   local check_cmd="git branch --list '$1'"
 400   msg=$( eval "$check_cmd" 2>&1 )
 401   if validate_ret_skip $? "Branch already exists."; then
 402     return
 403   fi
 404   if [ $DRY_RUN -eq 0 ]; then
 405     msg=$( eval "$cmd" 2>&1 )
 406     validate_ret $? "$msg"
 407   else
 408     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 409   fi
 410 }
 411
 412 # Switch to an existing branch, or checkout a new branch with the given
 413 # branch name.
 414 function switch_or_new_branch
 415 {
 416   local cmd="git rev-parse --verify '$1'"
 417   if [ $DRY_RUN -eq 0 ]; then
 418     # Call switch_branch if there is a branch, or new_branch if there is not
 419     msg=$( eval "$cmd" 2>&1 )
 420     RET=$?
 421     if [ $RET -eq 0 ]; then
 422       # Branch: (commit id)
 423       switch_branch "$1"
 424     elif [ $RET -eq 128 ]; then
 425       # Not a branch: "fatal: Needed a single revision"
 426       new_branch "$1"
 427     else
 428       # Unexpected return value
 429       validate_ret $RET "$msg"
 430     fi
 431   else
 432     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}, then depending on the result:"
 433     switch_branch "$1"
 434     new_branch "$1"
 435   fi
 436 }
 437
 438 # Set the upstream for branch to upstream.
 439 function set_upstream
 440 {
 441   # Note the argument order is swapped
 442   local cmd="git branch --set-upstream-to='$2' '$1'"
 443   printf "  %s Setting upstream for %s to %s..." "$MARKER" "$1" "$2"
 444   if [ $DRY_RUN -eq 0 ]; then
 445     msg=$( eval "$cmd" 2>&1 )
 446     validate_ret $? "$msg"
 447   else
 448     printf "\\n      %s\\n" "${IWTH}$cmd${CNRM}"
 449   fi
 450 }
 451
 452 ###############
 453 # Entry point #
 454 ###############
 455
 456 printf "%s Setting up the repository and remote %s\\n" "$MARKER" \
 457   "${BYEL}origin${CNRM}"
 458 # First, fetch the origin.
 459 ORIGIN_PARENT=$(dirname "$ORIGIN_PATH")
 460 make_directory "$ORIGIN_PARENT"
 461 # This is just cd with an error check
 462 goto_dir "$ORIGIN_PARENT"
 463
 464 # clone repository / origin remote
 465 clone_repo "$GIT_ORIGIN_PULL" "$TOR_MASTER_NAME"
 466 goto_dir "$ORIGIN_PATH"
 467 set_remote_push "origin" "$GIT_ORIGIN_PUSH"
 468
 469 # upstream remote, if different to origin
 470 if [ "$DEFAULT_UPSTREAM_REMOTE" != "origin" ]; then
 471   printf "%s Setting up remote %s\\n" "$MARKER" \
 472     "${BYEL}$DEFAULT_UPSTREAM_REMOTE${CNRM}"
 473   add_remote "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PULL"
 474   set_remote_push "$DEFAULT_UPSTREAM_REMOTE" "$GIT_UPSTREAM_PUSH"
 475   fetch_remote "$DEFAULT_UPSTREAM_REMOTE"
 476 fi
 477
 478 # GitHub remote
 479 printf "%s Setting up remote %s\\n" "$MARKER" "${BYEL}tor-github${CNRM}"
 480 # Add remote
 481 add_remote "tor-github" "$GITHUB_PULL"
 482 set_remote_push "tor-github" "$GITHUB_PUSH"
 483 # Add custom fetch for PRs
 484 set_tor_github_pr_fetch_config
 485 # Now fetch them all
 486 fetch_remote "tor-github"
 487
 488 # Extra remote
 489 if [ "$TOR_EXTRA_REMOTE_NAME" ]; then
 490   printf "%s Setting up remote %s\\n" "$MARKER" \
 491     "${BYEL}$TOR_EXTRA_REMOTE_NAME${CNRM}"
 492   # Add remote
 493   add_remote "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PULL"
 494   set_remote_push "$TOR_EXTRA_REMOTE_NAME" "$TOR_EXTRA_REMOTE_PUSH"
 495   # But leave it to the user to decide if they want to fetch it
 496   #fetch_remote "$TOR_EXTRA_REMOTE_NAME"
 497 fi
 498
 499 # Go over all configured worktree.
 500 for ((i=0; i<COUNT; i++)); do
 501   branch=${!WORKTREE[$i]:0:1}
 502   repo_path=${!WORKTREE[$i]:1:1}
 503
 504   printf "%s Handling branch %s\\n" "$MARKER" "${BYEL}$branch${CNRM}"
 505   # We cloned the repository, and main is the default branch
 506   if [ "$branch" = "main" ]; then
 507     if [ "$TOR_MASTER_NAME" != "main" ]; then
 508       # Set up a main branch link in the worktree directory
 509       make_symlink "$repo_path" "$GIT_PATH/$TOR_WKT_NAME/main"
 510     fi
 511   else
 512     # git makes worktree directories if they don't exist
 513     add_worktree "origin/$branch" "$repo_path"
 514   fi
 515   goto_dir "$repo_path"
 516   switch_or_new_branch "$branch"
 517   set_upstream "$branch" "origin/$branch"
 518 done
 519
 520 echo
 521 echo "Remember to copy the git hooks from tor/scripts/git/*.git-hook to"
 522 echo "$ORIGIN_PATH/.git/hooks/*"