cg-fetch

   1 #!/usr/bin/env bash
   2 #
   3 # Fetch changes from a remote branch to the local repository
   4 # Copyright (c) Petr Baudis, 2005.
   5 #
   6 # Takes the branch name as an argument. If no branch name was specified,
   7 # the default remote branch for the current branch (as selected by
   8 # `cg-switch -o`, that is the 'branch."currenthead".merge' option) is used.
   9 # If no such option exists, 'origin' is used.
  10 #
  11 # This will fetch the latest changes from a remote repository to the
  12 # corresponding branch in your local repository. Note that this operation
  13 # does not involve merging those changes to your own branch - that is being
  14 # done by the `cg-merge` command. `cg-update` exists to conveniently bundle
  15 # the act of fetching and merging to your working branch together.
  16 #
  17 # Before the first fetch, you have to tell Cogito about the remote branch.
  18 # This should be done by the `cg-branch-add` command. See its documentation
  19 # for the list of supported fetching protocols and other details. Note that
  20 # one exception to this is the 'origin' branch, which was set to the location
  21 # of the source repository if you created yours using the `cg-clone` command.
  22 #
  23 # Note that in the GIT newspeak, the operation being performed by cg-update
  24 # is now called 'pull', even though in the past and in many other version
  25 # control systems systems, 'pull' is the name for the operation performed by
  26 # `cg-fetch`. Please do not let this confuse you. (Cogito won't call this
  27 # 'update' operation 'pull', since about everyone but GIT and BK users uses
  28 # it in the 'fetch' meaning.)
  29 #
  30 # OPTIONS
  31 # -------
  32 # -f:: Force the complete fetch even if the heads are the same.
  33 #       Force the complete fetch even if the heads are the same.
  34 #
  35 # -v:: Enable verbosity
  36 #       Display more verbose output - most notably list all the files
  37 #       touched by the fetched changes. Use twice to get even more verbosity,
  38 #       that is raw progress information instead of the progress bar.
  39 #
  40 # ENVIRONMENT
  41 # -----------
  42 # RSYNC::
  43 #       The command to invoke when we want to call the rsync tool (only used
  44 #       when fetching over the rsync protocol). Defaults to 'rsync'.
  45 #
  46 # RSYNC_FLAGS::
  47 #       Additional flags to be passed to the rsync tool when fetching over
  48 #       the rsync protocol.
  49
  50 # Testsuite: Largely covered (t91xx testsuite family, incomplete coverage)
  51
  52
  53 USAGE="cg-fetch [-f] [-v] [BRANCH_NAME]"
  54 _git_wc_unneeded=1
  55
  56 . "${COGITO_LIB}"cg-Xlib || exit 1
  57
  58
  59 fetch_progress()
  60 {
  61         [ $verbose -ge 2 ] && exec cat
  62         if [ -t 1 ]; then
  63                 exec "${COGITO_LIB}"cg-Xfetchprogress "$_git_objects"
  64         else
  65                 exec cat >/dev/null
  66         fi
  67 }
  68
  69 show_changes_summary()
  70 {
  71         local orig_head="$1"
  72         local new_head="$2"
  73         if [ ! "$orig_head" ]; then
  74                 echo "New branch: $new_head"
  75
  76         elif [ "$orig_head" != "$new_head" ]; then
  77                 echo "Tree change: $orig_head..$new_head"
  78                 [ $verbose -ge 1 ] && git-diff-tree --abbrev -r "$(cg-object-id -t "$orig_head")" "$(cg-object-id -t "$new_head")"
  79         else
  80                 echo "Up to date."
  81         fi
  82 }
  83
  84 initial_done()
  85 {
  86         rm -f ___
  87         git-update-ref "refs/heads/master" "$(get_ref "refs/heads/origin")" ||
  88                 die "initial checkout failed"
  89         if [ -s "$_git/info/cg-fetch-initial-wcless" ]; then
  90                 rm "$_git/info/cg-fetch-initial-wcless"
  91         else
  92                 git-read-tree HEAD &&
  93                 git-checkout-index -a &&
  94                 git-update-index --refresh ||
  95                 die "initial checkout failed"
  96         fi
  97         rm "$_git/info/cg-fetch-initial"
  98 }
  99
 100
 101 get_rsync()
 102 {
 103         [ "$1" = "-b" ] && shift
 104
 105         redir=
 106         if [ "$1" = "-i" ]; then # ignore-errors
 107                 redir="2>/dev/null"
 108                 shift
 109         fi
 110
 111         filter="cat"
 112         if [ "$1" = "-s" ]; then # subsequent
 113                 # We already saw the MOTD, thank you very much.
 114                 filter="grep -v ^MOTD:"
 115                 shift
 116         fi
 117
 118         appenduri=
 119         if [ "$1" = "-d" ]; then # directory
 120                 appenduri="/." # CowboyNeal
 121                 shift
 122         fi
 123
 124         echo "${RSYNC:-rsync}" $RSYNC_FLAGS -v --partial -Lr \
 125                 "$1$appenduri" "$2$appenduri" $redir
 126         eval '"${RSYNC:-rsync}"' $RSYNC_FLAGS -v --partial -Lr \
 127                 '"$1$appenduri"' '"$2$appenduri"' $redir | $filter
 128         return ${PIPESTATUS[0]}
 129 }
 130
 131 fetch_rsync_verify()
 132 {
 133         if [ $verbose -ge 2 ]; then
 134                 # We must not pipe to prevent buffered I/O
 135                 get_rsync -s -d "$2/objects" "$_git_objects"
 136         else
 137                 get_rsync -s -d "$2/objects" "$_git_objects" | fetch_progress
 138         fi
 139
 140         ret=${PIPESTATUS[0]}
 141         if [ "$3" ] && [ "$ret" -eq "0" ]; then
 142                 if [ "$orig_head" ]; then
 143                         git-rev-list --objects $new_head ^$orig_head |
 144                                 while read obj type; do
 145                                         git-cat-file -t $obj >/dev/null || exit $?
 146                                 done ||
 147                         die "rsync fetch incomplete, some objects missing"
 148                 fi
 149                 git-update-ref "refs/$3" "$1"
 150         fi
 151         return $ret
 152 }
 153
 154
 155 fetch_rsync()
 156 {
 157         if [ x"$1" = x"--stdin" ]; then
 158                 while read c w; do
 159                         git-update-ref "refs/$w" "$c"
 160                 done
 161         else
 162                 fetch_rsync_verify "$1" "$2" "$3"
 163         fi
 164 }
 165
 166 get_http()
 167 {
 168         [ "$1" = "-b" ] && shift
 169         [ "$1" = "-i" ] && shift
 170         [ "$1" = "-s" ] && shift
 171         [ "$1" = "-d" ] && die "INTERNAL ERROR: HTTP recursive not implemented"
 172
 173         src="$1"
 174         dest="$2"
 175
 176         curl_extra_args=
 177         [ "$GIT_SSL_NO_VERIFY" ] && curl_extra_args="-k"
 178         curl -nsfL $curl_extra_args -o "$dest" "$src"
 179 }
 180
 181 fetch_http()
 182 {
 183         whead=
 184         [ "$3" ] && whead="-w $3"
 185         (git-http-fetch -a -v $whead $recovery "$1" "$2/" 2>&1 /dev/null) | fetch_progress
 186         return ${PIPESTATUS[0]}
 187 }
 188
 189
 190 get_local()
 191 {
 192         #cp_flags_l="-v"
 193         cp_flags_l=
 194         if [ "$1" = "-b" ]; then
 195                 # Dereference symlinks
 196                 cp_flags_l="$cp_flags_l -L"
 197                 shift
 198         else
 199                 cp_flags_l="$cp_flags_l -pRP"
 200         fi
 201
 202         [ "$1" = "-i" ] && shift
 203         [ "$1" = "-s" ] && shift
 204         [ "$1" = "-d" ] && die "INTERNAL ERROR: local-fetch recursive not implemented"
 205
 206         src="$1"
 207         dest="$2"
 208
 209         cp $cp_flags_l "$src" "$dest"
 210 }
 211
 212 fetch_local()
 213 {
 214         whead=
 215         [ "$3" ] && whead="-w $3"
 216         (git-local-fetch -a -l -v $whead $recovery "$1" "$2" 2>&1 /dev/null) | fetch_progress
 217         return ${PIPESTATUS[0]}
 218 }
 219
 220
 221 fetch_tags()
 222 {
 223         echo -n "Fetching tags... "
 224
 225         # FIXME: Warn about conflicting tag names?
 226
 227         if [ "$get" = "get_rsync" ]; then
 228                 echo
 229                 warn "WHAT I'M DOING NOW IS RACY AND BROKEN IF YOU USE PACKED REFS!"
 230                 warn "Please switch from rsync to something else."
 231                 [ -d "$_git/refs/tags" ] || mkdir -p "$_git/refs/tags"
 232                 if ! $get -i -s -d "$uri/refs/tags" "$_git/refs/tags"; then
 233                         echo "unable to get tags list (non-fatal)" >&2
 234                         return $?
 235                 fi
 236         fi
 237
 238         last_tag=""
 239         (
 240         git-ls-remote --tags "$uri" |
 241                 # SHA1 refs/tags/v0.99.8^{} --> SHA1 tags/v0.99.8
 242                 # where SHA1 is the object v0.99.8 tag points at.
 243                 sed -n -e 's:\([^       ]\)     refs/\(tags/.*\)^{}$:\1 \2:p' \
 244                        -e 's:\([^       ]\)     refs/\(tags/.*\)$:\1 \2:p' | \
 245                 while read sha1 tagname; do
 246                         # Do we have the tag itself?
 247                         exists_ref "refs/$tagname" && continue
 248                         # Do we have the object pointed at by the tag?
 249                         git-cat-file -t "$sha1" >/dev/null 2>&1 || continue
 250
 251                         # In common case we will get both "normal" and ^{} entries.
 252                         # Filter out the dupes.
 253                         [ "$last_tag" = "$tagname" ] && continue
 254                         last_tag="$tagname"
 255
 256                         # if so, fetch the tag -- which should be
 257                         # a cheap operation -- to complete the chain.
 258                         echo -n "${tagname#tags/} " >&3
 259                         echo -e "$tagname"\\t"$tagname"
 260                 done |
 261                 sort | uniq | $fetch --stdin "$uri"
 262         ) 3>&1
 263         echo
 264         if [ "${PIPESTATUS[0]}" -ne 0 -o "$?" -ne 0 ]; then
 265                 echo "unable to fetch tags (non-fatal)" >&2
 266         fi
 267         return 0
 268 }
 269
 270
 271 recovery=
 272 verbose=0
 273 while optparse; do
 274         if optparse -f; then
 275                 # When forcing, let the fetch tools make more extensive
 276                 # walk over the dependency tree with --recover.
 277                 recovery=--recover
 278         elif optparse -v; then
 279                 verbose=$((verbose+1))
 280         else
 281                 optfail
 282         fi
 283 done
 284
 285 name="${ARGS[0]}"
 286
 287 [ "$name" ] || name="$(choose_origin branches "where to fetch from?")" || exit 1
 288 uri=$(cat "$_git/branches/$name" 2>/dev/null) || die "unknown branch: $name"
 289
 290 rembranch=
 291 if echo "$uri" | grep -q '#'; then
 292         rembranch=$(echo "$uri" | cut -d '#' -f 2)
 293         uri=$(echo "$uri" | cut -d '#' -f 1)
 294 fi
 295
 296 if [ "$_git_no_wc" ]; then
 297         [ -s "$_git/info/cg-fetch-initial" ] && [ ! -s "$_git/info/cg-fetch-initial-wcless" ] &&
 298                 die "you must run the initial cg-fetch from the working copy root directory"
 299 fi
 300
 301 # Some other process with the same pid might appear, that's why
 302 # we won't die but rather let the user check quickly.
 303 dirtyfile="$_git/info/cg-fetch-$(echo "$name" | sed -e 's/\//-.-/g')-dirty"
 304 if [ -s "$dirtyfile" ]; then
 305         kill -0 $(cat "$dirtyfile") 2>/dev/null && \
 306                 warn "aren't you fetching $name twice at once? (waiting 10s)" && \
 307                 sleep 10
 308         if [ -s "$_git/info/cg-fetch-initial" ]; then
 309                 echo "Recovering from a previously interrupted initial clone..."
 310         else
 311                 echo "Recovering from a previously interrupted fetch..."
 312         fi
 313         recovery=--recover
 314 fi
 315 mkdir -p "$_git/info"
 316 echo $$ > "$dirtyfile"
 317
 318
 319 orig_head="$(get_ref "refs/heads/$name")" || : may stay empty
 320
 321
 322 packed_transport=
 323
 324 if echo "$uri" | grep -q "^\(https\?\|ftp\)://"; then
 325         get=get_http
 326         fetch=fetch_http
 327 elif echo "$uri" | grep -q "^git+ssh://"; then
 328         packed_transport=ssh
 329 elif echo "$uri" | grep -q "^git://"; then
 330         packed_transport=git
 331 elif echo "$uri" | grep -q "^rsync://"; then
 332         echo "WARNING: The rsync access method is DEPRECATED and will be REMOVED in the future!" >&2
 333         get=get_rsync
 334         fetch=fetch_rsync
 335 elif echo "$uri" | grep -q ":"; then
 336         echo "WARNING: I guessed the host:path syntax was used and fell back to the git+ssh protocol." >&2
 337         echo "WARNING: The host:path syntax is evil because it is implicit. Please just use a URI." >&2
 338         packed_transport=ssh
 339 else
 340         [ -d "$uri/.git" ] && uri="$uri/.git"
 341         [ -d "$uri" ] || die "repository not found"
 342         get=get_local
 343         fetch=fetch_local
 344
 345         # Perhaps the object database is shared
 346         symlinked=
 347         is_same_repo "$_git_objects" "$uri/objects" && symlinked=1
 348
 349         # See if we can hardlink and add "-l" to cp flags.
 350         can_hardlink=
 351         sample_file="$(find "$uri" -type f -print | head -n 1)"
 352         rm -f "$_git/.,,lntest"
 353         if cp -fl "$sample_file" "$_git/.,,lntest" 2>/dev/null; then
 354                 can_hardlink=l
 355                 echo "Using hard links"
 356         else
 357                 echo "Hard links don't work - using copy"
 358         fi
 359         rm -f "$_git/.,,lntest"
 360 fi
 361
 362
 363 if [ "$packed_transport" ]; then
 364         # This is a really special case.
 365         [ "$rembranch" ] || rembranch="HEAD"
 366
 367         cloneorfetch= #fetch
 368         [ -s "$_git/info/cg-fetch-initial" ] && cloneorfetch=-k #clone
 369
 370         rm -f "$_git/info/cg-fetch-earlydie"
 371
 372         fetch_pack_recorder () {
 373                 while read sha1 remote_name; do
 374                         [ "$sha1" = "failed" ] && die "$2"
 375                         ref="$1"; [ "$ref" ] || ref="$remote_name"
 376                         git-update-ref "$ref" "$sha1"
 377                 done
 378         }
 379         echo "Fetching pack (head and objects)..."
 380         ( git-fetch-pack $cloneorfetch "$uri" "$rembranch" ||
 381           echo "failed" "$rembranch" ) |
 382                 fetch_pack_recorder "refs/heads/$name" "fetching pack failed" ||
 383                 exit
 384
 385         record_tags_to_fetch () {
 386                 ( cut -f 1 | tr '\n' '\0' |
 387                         xargs -0 git-fetch-pack $cloneorfetch "$uri" ||
 388                   echo "failed" "$rembranch" ) |
 389
 390                 fetch_pack_recorder "" "unable to retrieve tags (non-fatal)"
 391         }
 392         fetch=record_tags_to_fetch
 393         fetch_tags
 394
 395         rm "$dirtyfile"
 396         show_changes_summary "$orig_head" "$(cg-object-id "$name")"
 397         [ -s "$_git/info/cg-fetch-initial" ] && initial_done
 398         exit 0
 399 fi
 400
 401
 402 ### Behold, the fetch itself
 403
 404 ## Grab the head
 405 echo "Fetching head..."
 406
 407 tmpname="$(mktemp -t githead.XXXXXX)"
 408 cleanup () {
 409         rm "$tmpname"
 410 }
 411 cleanup_trap "cleanup"
 412
 413 if [ "$rembranch" ]; then
 414         $get -i "$uri/refs/heads/$rembranch" "$tmpname" ||
 415                 die "unable to get the head pointer of branch $rembranch"
 416 else
 417         $get -b "$uri/HEAD" "$tmpname" ||
 418                 die "unable to get the HEAD branch"
 419 fi
 420
 421 new_head="$(cat "$tmpname")"
 422 if [ "${new_head#ref:}" != "$new_head" ]; then
 423         new_head="$(echo "$new_head" | sed 's/^ref: *//')"
 424         $get -i "$uri/$new_head" "$tmpname" ||
 425                 die "unable to get the head pointer of branch $new_head (referenced by HEAD)"
 426         new_head="$(cat "$tmpname")"
 427 fi
 428
 429 rm -f "$_git/info/cg-fetch-earlydie"
 430
 431 echo "Fetching objects..."
 432 ## Fetch the objects
 433 if ! [ "$symlinked" ]; then
 434         if [ "$recovery" -o "$orig_head" != "$new_head" ]; then
 435                 [ -d "$_git_objects" ] || mkdir -p "$_git_objects"
 436                 $fetch "$(cat "$tmpname")" "$uri" "heads/$name" || die "objects fetch failed"
 437         fi
 438 else
 439         git-update-ref "refs/heads/$name" "$(cat "$tmpname")"
 440 fi
 441
 442 rm "$tmpname"
 443 cleanup_trap ""
 444
 445 ## Fetch the tags
 446 ret=0
 447 if ! fetch_tags; then
 448         ret=$?
 449 fi
 450
 451 rm "$dirtyfile"
 452 show_changes_summary "$orig_head" "$new_head"
 453 [ -s "$_git/info/cg-fetch-initial" ] && initial_done
 454 exit $ret