pkgs/build-support/fetchgit/nix-prefetch-git

   1 #! /usr/bin/env bash
   2
   3 set -e -o pipefail
   4
   5 url=
   6 rev=
   7 expHash=
   8 hashType=$NIX_HASH_ALGO
   9 deepClone=$NIX_PREFETCH_GIT_DEEP_CLONE
  10 leaveDotGit=$NIX_PREFETCH_GIT_LEAVE_DOT_GIT
  11 fetchSubmodules=
  12 fetchLFS=
  13 builder=
  14 branchName=$NIX_PREFETCH_GIT_BRANCH_NAME
  15
  16 # ENV params
  17 out=${out:-}
  18 http_proxy=${http_proxy:-}
  19
  20 # NOTE: use of NIX_GIT_SSL_CAINFO is for backwards compatibility; NIX_SSL_CERT_FILE is preferred
  21 # as of PR#303307
  22 GIT_SSL_CAINFO=${NIX_GIT_SSL_CAINFO:-$NIX_SSL_CERT_FILE}
  23
  24 # populated by clone_user_rev()
  25 fullRev=
  26 humanReadableRev=
  27 commitDate=
  28 commitDateStrict8601=
  29
  30 if test -n "$deepClone"; then
  31     deepClone=true
  32 else
  33     deepClone=
  34 fi
  35
  36 if test "$leaveDotGit" != 1; then
  37     leaveDotGit=
  38 else
  39     leaveDotGit=true
  40 fi
  41
  42 usage(){
  43     echo  >&2 "syntax: nix-prefetch-git [options] [URL [REVISION [EXPECTED-HASH]]]
  44
  45 Options:
  46       --out path      Path where the output would be stored.
  47       --url url       Any url understood by 'git clone'.
  48       --rev ref       Any sha1 or references (such as refs/heads/master)
  49       --hash h        Expected hash.
  50       --branch-name   Branch name to check out into
  51       --sparse-checkout Only fetch and checkout part of the repository.
  52       --non-cone-mode Use non-cone mode for sparse checkouts.
  53       --deepClone     Clone the entire repository.
  54       --no-deepClone  Make a shallow clone of just the required ref.
  55       --leave-dotGit  Keep the .git directories.
  56       --fetch-lfs     Fetch git Large File Storage (LFS) files.
  57       --fetch-submodules Fetch submodules.
  58       --builder       Clone as fetchgit does, but url, rev, and out option are mandatory.
  59       --quiet         Only print the final json summary.
  60 "
  61     exit 1
  62 }
  63
  64 # some git commands print to stdout, which would contaminate our JSON output
  65 clean_git(){
  66     git "$@" >&2
  67 }
  68
  69 argi=0
  70 argfun=""
  71 for arg; do
  72     if test -z "$argfun"; then
  73         case $arg in
  74             --out) argfun=set_out;;
  75             --url) argfun=set_url;;
  76             --rev) argfun=set_rev;;
  77             --hash) argfun=set_hashType;;
  78             --branch-name) argfun=set_branchName;;
  79             --deepClone) deepClone=true;;
  80             --sparse-checkout) argfun=set_sparseCheckout;;
  81             --non-cone-mode) nonConeMode=true;;
  82             --quiet) QUIET=true;;
  83             --no-deepClone) deepClone=;;
  84             --leave-dotGit) leaveDotGit=true;;
  85             --fetch-lfs) fetchLFS=true;;
  86             --fetch-submodules) fetchSubmodules=true;;
  87             --builder) builder=true;;
  88             -h|--help) usage; exit;;
  89             *)
  90                 : $((++argi))
  91                 case $argi in
  92                     1) url=$arg;;
  93                     2) rev=$arg;;
  94                     3) expHash=$arg;;
  95                     *) exit 1;;
  96                 esac
  97                 ;;
  98         esac
  99     else
 100         case $argfun in
 101             set_*)
 102                 var=${argfun#set_}
 103                 eval "$var=$(printf %q "$arg")"
 104                 ;;
 105         esac
 106         argfun=""
 107     fi
 108 done
 109
 110 if test -z "$url"; then
 111     usage
 112 fi
 113
 114
 115 init_remote(){
 116     local url=$1
 117     clean_git init --initial-branch=master
 118     clean_git remote add origin "$url"
 119     if [ -n "$sparseCheckout" ]; then
 120         git config remote.origin.partialclonefilter "blob:none"
 121         echo "$sparseCheckout" | git sparse-checkout set --stdin ${nonConeMode:+--no-cone}
 122     fi
 123     ( [ -n "$http_proxy" ] && clean_git config --global http.proxy "$http_proxy" ) || true
 124 }
 125
 126 # Return the reference of an hash if it exists on the remote repository.
 127 ref_from_hash(){
 128     local hash=$1
 129     git ls-remote origin | sed -n "\,$hash\t, { s,\(.*\)\t\(.*\),\2,; p; q}"
 130 }
 131
 132 # Return the hash of a reference if it exists on the remote repository.
 133 hash_from_ref(){
 134     local ref=$1
 135     git ls-remote origin | sed -n "\,\t$ref, { s,\(.*\)\t\(.*\),\1,; p; q}"
 136 }
 137
 138 # Returns a name based on the url and reference
 139 #
 140 # This function needs to be in sync with nix's fetchgit implementation
 141 # of urlToName() to re-use the same nix store paths.
 142 url_to_name(){
 143     local url=$1
 144     local ref=$2
 145     local base
 146     base=$(basename "$url" .git | cut -d: -f2)
 147
 148     if [[ $ref =~ ^[a-z0-9]+$ ]]; then
 149         echo "$base-${ref:0:7}"
 150     else
 151         echo "$base"
 152     fi
 153 }
 154
 155 # Fetch and checkout the right sha1
 156 checkout_hash(){
 157     local hash="$1"
 158     local ref="$2"
 159
 160     if test -z "$hash"; then
 161         hash=$(hash_from_ref "$ref")
 162     fi
 163
 164     [[ -z "$deepClone" ]] && \
 165     clean_git fetch ${builder:+--progress} --depth=1 origin "$hash" || \
 166     clean_git fetch -t ${builder:+--progress} origin || return 1
 167
 168     local object_type=$(git cat-file -t "$hash")
 169     if [[ "$object_type" == "commit" || "$object_type" == "tag" ]]; then
 170         clean_git checkout -b "$branchName" "$hash" || return 1
 171     elif [[ "$object_type" == "tree" ]]; then
 172         clean_git config user.email "nix-prefetch-git@localhost"
 173         clean_git config user.name "nix-prefetch-git"
 174         local commit_id=$(git commit-tree "$hash" -m "Commit created from tree hash $hash")
 175         clean_git checkout -b "$branchName" "$commit_id" || return 1
 176     else
 177         echo "Unrecognized git object type: $object_type"
 178         return 1
 179     fi
 180 }
 181
 182 # Fetch only a branch/tag and checkout it.
 183 checkout_ref(){
 184     local hash="$1"
 185     local ref="$2"
 186
 187     if [[ -n "$deepClone" ]]; then
 188         # The caller explicitly asked for a deep clone.  Deep clones
 189         # allow "git describe" and similar tools to work.  See
 190         # https://marc.info/?l=nix-dev&m=139641582514772
 191         # for a discussion.
 192         return 1
 193     fi
 194
 195     if test -z "$ref"; then
 196         ref=$(ref_from_hash "$hash")
 197     fi
 198
 199     if test -n "$ref"; then
 200         # --depth option is ignored on http repository.
 201         clean_git fetch ${builder:+--progress} --depth 1 origin +"$ref" || return 1
 202         clean_git checkout -b "$branchName" FETCH_HEAD || return 1
 203     else
 204         return 1
 205     fi
 206 }
 207
 208 # Update submodules
 209 init_submodules(){
 210     # shallow with leaveDotGit will change hashes
 211     [[ -z "$deepClone" ]] && [[ -z "$leaveDotGit" ]] && \
 212     clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} --progress --depth 1 || \
 213     clean_git submodule update --init --recursive -j ${NIX_BUILD_CORES:-1} --progress
 214 }
 215
 216 clone(){
 217     local top=$PWD
 218     local dir="$1"
 219     local url="$2"
 220     local hash="$3"
 221     local ref="$4"
 222
 223     cd "$dir"
 224
 225     # Initialize the repository.
 226     init_remote "$url"
 227
 228     # Download data from the repository.
 229     checkout_ref "$hash" "$ref" ||
 230     checkout_hash "$hash" "$ref" || (
 231         echo 1>&2 "Unable to checkout $hash$ref from $url."
 232         exit 1
 233     )
 234
 235     # Checkout linked sources.
 236     if test -n "$fetchSubmodules"; then
 237         init_submodules
 238     fi
 239
 240     if [ -z "$builder" ] && [ -f .topdeps ]; then
 241         if tg help &>/dev/null; then
 242             echo "populating TopGit branches..."
 243             tg remote --populate origin
 244         else
 245             echo "WARNING: would populate TopGit branches but TopGit is not available" >&2
 246             echo "WARNING: install TopGit to fix the problem" >&2
 247         fi
 248     fi
 249
 250     cd "$top"
 251 }
 252
 253 # Remove all remote branches, remove tags not reachable from HEAD, do a full
 254 # repack and then garbage collect unreferenced objects.
 255 make_deterministic_repo(){
 256     local repo="$1"
 257
 258     # run in sub-shell to not touch current working directory
 259     (
 260     cd "$repo"
 261     # Remove files that contain timestamps or otherwise have non-deterministic
 262     # properties.
 263     if [ -f .git ]; then
 264         local dotgit_content=$(<.git)
 265         local dotgit_dir="${dotgit_content#gitdir: }"
 266     else
 267         local dotgit_dir=".git"
 268     fi
 269     pushd "$dotgit_dir" >/dev/null
 270         rm -rf logs/ hooks/ index FETCH_HEAD ORIG_HEAD refs/remotes/origin/HEAD config
 271     popd >/dev/null
 272     # Remove all remote branches.
 273     git branch -r | while read -r branch; do
 274         clean_git branch -rD "$branch"
 275     done
 276
 277     # Remove tags not reachable from HEAD. If we're exactly on a tag, don't
 278     # delete it.
 279     maybe_tag=$(git tag --points-at HEAD)
 280     git tag --contains HEAD | while read -r tag; do
 281         if [ "$tag" != "$maybe_tag" ]; then
 282             clean_git tag -d "$tag"
 283         fi
 284     done
 285
 286     # Do a full repack. Must run single-threaded, or else we lose determinism.
 287     clean_git config pack.threads 1
 288     clean_git repack -A -d -f
 289     rm -f "$dotgit_dir/config"
 290
 291     # Garbage collect unreferenced objects.
 292     # Note: --keep-largest-pack prevents non-deterministic ordering of packs
 293     #   listed in .git/objects/info/packs by only using a single pack
 294     clean_git gc --prune=all --keep-largest-pack
 295     )
 296 }
 297
 298
 299 clone_user_rev() {
 300     local dir="$1"
 301     local url="$2"
 302     local rev="${3:-HEAD}"
 303
 304     if [ -n "$fetchLFS" ]; then
 305         clean_git lfs install
 306     fi
 307
 308     # Perform the checkout.
 309     case "$rev" in
 310         HEAD|refs/*)
 311             clone "$dir" "$url" "" "$rev" 1>&2;;
 312         *)
 313             if test -z "$(echo "$rev" | tr -d 0123456789abcdef)"; then
 314                 clone "$dir" "$url" "$rev" "" 1>&2
 315             else
 316                 # if revision is not hexadecimal it might be a tag
 317                 clone "$dir" "$url" "" "refs/tags/$rev" 1>&2
 318             fi;;
 319     esac
 320
 321     pushd "$dir" >/dev/null
 322     fullRev=$( (git rev-parse "$rev" 2>/dev/null || git rev-parse "refs/heads/$branchName") | tail -n1)
 323     humanReadableRev=$(git describe "$fullRev" 2> /dev/null || git describe --tags "$fullRev" 2> /dev/null || echo -- none --)
 324     commitDate=$(git show -1 --no-patch --pretty=%ci "$fullRev")
 325     commitDateStrict8601=$(git show -1 --no-patch --pretty=%cI "$fullRev")
 326     popd >/dev/null
 327
 328     # Allow doing additional processing before .git removal
 329     eval "$NIX_PREFETCH_GIT_CHECKOUT_HOOK"
 330     if test -z "$leaveDotGit"; then
 331         echo "removing \`.git'..." >&2
 332         find "$dir" -name .git -print0 | xargs -0 rm -rf
 333     else
 334         find "$dir" -name .git | while read -r gitdir; do
 335             make_deterministic_repo "$(readlink -f "$(dirname "$gitdir")")"
 336         done
 337     fi
 338 }
 339
 340 exit_handlers=()
 341
 342 run_exit_handlers() {
 343     exit_status=$?
 344     for handler in "${exit_handlers[@]}"; do
 345         eval "$handler $exit_status"
 346     done
 347 }
 348
 349 trap run_exit_handlers EXIT
 350
 351 quiet_exit_handler() {
 352     exec 2>&3 3>&-
 353     if [ $1 -ne 0 ]; then
 354         cat "$errfile" >&2
 355     fi
 356     rm -f "$errfile"
 357 }
 358
 359 quiet_mode() {
 360     errfile="$(mktemp "${TMPDIR:-/tmp}/git-checkout-err-XXXXXXXX")"
 361     exit_handlers+=(quiet_exit_handler)
 362     exec 3>&2 2>"$errfile"
 363 }
 364
 365 json_escape() {
 366     local s="$1"
 367     s="${s//\\/\\\\}" # \
 368     s="${s//\"/\\\"}" # "
 369     s="${s//^H/\\\b}" # \b (backspace)
 370     s="${s//^L/\\\f}" # \f (form feed)
 371     s="${s//
 372 /\\\n}" # \n (newline)
 373     s="${s//^M/\\\r}" # \r (carriage return)
 374     s="${s//   /\\t}" # \t (tab)
 375     echo "$s"
 376 }
 377
 378 print_results() {
 379     hash="$1"
 380     if ! test -n "$QUIET"; then
 381         echo "" >&2
 382         echo "git revision is $fullRev" >&2
 383         if test -n "$finalPath"; then
 384             echo "path is $finalPath" >&2
 385         fi
 386         echo "git human-readable version is $humanReadableRev" >&2
 387         echo "Commit date is $commitDate" >&2
 388         if test -n "$hash"; then
 389             echo "hash is $hash" >&2
 390         fi
 391     fi
 392     if test -n "$hash"; then
 393         cat <<EOF
 394 {
 395   "url": "$(json_escape "$url")",
 396   "rev": "$(json_escape "$fullRev")",
 397   "date": "$(json_escape "$commitDateStrict8601")",
 398   "path": "$(json_escape "$finalPath")",
 399   "$(json_escape "$hashType")": "$(json_escape "$hash")",
 400   "hash": "$(nix-hash --to-sri --type $hashType $hash)",
 401   "fetchLFS": $([[ -n "$fetchLFS" ]] && echo true || echo false),
 402   "fetchSubmodules": $([[ -n "$fetchSubmodules" ]] && echo true || echo false),
 403   "deepClone": $([[ -n "$deepClone" ]] && echo true || echo false),
 404   "leaveDotGit": $([[ -n "$leaveDotGit" ]] && echo true || echo false)
 405 }
 406 EOF
 407     fi
 408 }
 409
 410 remove_tmpPath() {
 411     rm -rf "$tmpPath"
 412 }
 413
 414 remove_tmpHomePath() {
 415     chmod -R u+w "$tmpHomePath"
 416     rm -rf "$tmpHomePath"
 417 }
 418
 419 if test -n "$QUIET"; then
 420     quiet_mode
 421 fi
 422
 423 if test -z "$branchName"; then
 424     branchName=fetchgit
 425 fi
 426
 427 tmpHomePath="$(mktemp -d "${TMPDIR:-/tmp}/nix-prefetch-git-tmp-home-XXXXXXXXXX")"
 428 exit_handlers+=(remove_tmpHomePath)
 429 ln -s "${NETRC:-$HOME/.netrc}" "$tmpHomePath/.netrc"
 430 HOME="$tmpHomePath"
 431 unset XDG_CONFIG_HOME
 432 export GIT_CONFIG_NOSYSTEM=1
 433
 434 if test -n "$builder"; then
 435     test -n "$out" -a -n "$url" -a -n "$rev" || usage
 436     mkdir -p "$out"
 437     clone_user_rev "$out" "$url" "$rev"
 438 else
 439     if test -z "$hashType"; then
 440         hashType=sha256
 441     fi
 442
 443     # If the hash was given, a file with that hash may already be in the
 444     # store.
 445     if test -n "$expHash"; then
 446         finalPath=$(nix-store --print-fixed-path --recursive "$hashType" "$expHash" "$(url_to_name "$url" "$rev")")
 447         if ! nix-store --check-validity "$finalPath" 2> /dev/null; then
 448             finalPath=
 449         fi
 450         hash=$expHash
 451     fi
 452
 453     # If we don't know the hash or a path with that hash doesn't exist,
 454     # download the file and add it to the store.
 455     if test -z "$finalPath"; then
 456
 457         tmpPath="$(mktemp -d "${TMPDIR:-/tmp}/git-checkout-tmp-XXXXXXXX")"
 458         exit_handlers+=(remove_tmpPath)
 459
 460         tmpFile="$tmpPath/$(url_to_name "$url" "$rev")"
 461         mkdir -p "$tmpFile"
 462
 463         # Perform the checkout.
 464         clone_user_rev "$tmpFile" "$url" "$rev"
 465
 466         # Compute the hash.
 467         hash=$(nix-hash --type $hashType --base32 "$tmpFile")
 468
 469         # Add the downloaded file to the Nix store.
 470         finalPath=$(nix-store --add-fixed --recursive "$hashType" "$tmpFile")
 471
 472         if test -n "$expHash" -a "$expHash" != "$hash"; then
 473             echo "hash mismatch for URL \`$url'. Got \`$hash'; expected \`$expHash'." >&2
 474             exit 1
 475         fi
 476     fi
 477
 478     print_results "$hash"
 479
 480     if test -n "$PRINT_PATH"; then
 481         echo "$finalPath"
 482     fi
 483 fi