maintainers/scripts/bootstrap-files/refresh-tarballs.bash

   1 #!/usr/bin/env nix-shell
   2 #! nix-shell --pure
   3 #! nix-shell -i bash
   4 #! nix-shell -p curl cacert
   5 #! nix-shell -p git
   6 #! nix-shell -p nix
   7 #! nix-shell -p jq
   8
   9 set -o pipefail
  10
  11 # How the refresher works:
  12 #
  13 # For a given list of <targets>:
  14 # 1. fetch latest successful '.build` job
  15 # 2. fetch oldest evaluation that contained that '.build', extract nixpkgs commit
  16 # 3. fetch all the `.build` artifacts from '$out/on-server/' directory
  17 # 4. calculate hashes and craft the commit message with the details on
  18 #    how to upload the result to 'tarballs.nixos.org'
  19
  20 scratch_dir=$(mktemp -d)
  21 trap 'rm -rf -- "${scratch_dir}"' EXIT
  22
  23 usage() {
  24     cat >&2 <<EOF
  25 Usage:
  26     $0 [ --commit ] --targets=<target>[,<target>,...]
  27
  28     The tool must be ran from the root directory of 'nixpkgs' repository.
  29
  30 Synopsis:
  31     'refresh-tarballs.bash' script fetches latest bootstrapFiles built
  32     by hydra, registers them in 'nixpkgs' and provides commands to
  33     upload seed files to 'tarballs.nixos.org'.
  34
  35     This is usually done in the following cases:
  36
  37     1. Single target fix: current bootstrap files for a single target
  38        are problematic for some reason (target-specific bug). In this
  39        case we can refresh just that target as:
  40
  41        \$ $0 --commit --targets=i686-unknown-linux-gnu
  42
  43     2. Routine refresh: all bootstrap files should be refreshed to avoid
  44        debugging problems that only occur on very old binaries.
  45
  46        \$ $0 --commit --all-targets
  47
  48 To get help on uploading refreshed binaries to 'tarballs.nixos.org'
  49 please have a look at <maintainers/scripts/bootstrap-files/README.md>.
  50 EOF
  51     exit 1
  52 }
  53
  54 # log helpers
  55
  56 die() {
  57     echo "ERROR: $*" >&2
  58     exit 1
  59 }
  60
  61 info() {
  62     echo "INFO: $*" >&2
  63 }
  64
  65 [[ ${#@} -eq 0 ]] && usage
  66
  67 # known targets
  68
  69 NATIVE_TARGETS=(
  70     aarch64-unknown-linux-gnu
  71     aarch64-unknown-linux-musl
  72     i686-unknown-linux-gnu
  73     x86_64-unknown-linux-gnu
  74     x86_64-unknown-linux-musl
  75     aarch64-apple-darwin
  76     x86_64-apple-darwin
  77 )
  78
  79 is_native() {
  80     local t target=$1
  81     for t in "${NATIVE_TARGETS[@]}"; do
  82         [[ $t == $target ]] && return 0
  83     done
  84     return 1
  85 }
  86
  87 CROSS_TARGETS=(
  88     armv5tel-unknown-linux-gnueabi
  89     armv6l-unknown-linux-gnueabihf
  90     armv6l-unknown-linux-musleabihf
  91     armv7l-unknown-linux-gnueabihf
  92     mips64el-unknown-linux-gnuabi64
  93     mips64el-unknown-linux-gnuabin32
  94     mipsel-unknown-linux-gnu
  95     powerpc64-unknown-linux-gnuabielfv2
  96     powerpc64le-unknown-linux-gnu
  97     riscv64-unknown-linux-gnu
  98     s390x-unknown-linux-gnu
  99     x86_64-unknown-freebsd
 100 )
 101
 102 is_cross() {
 103     local t target=$1
 104     for t in "${CROSS_TARGETS[@]}"; do
 105         [[ $t == $target ]] && return 0
 106     done
 107     return 1
 108 }
 109
 110 nar_sri_get() {
 111     local restore_path store_path
 112     ((${#@} != 2)) && die "nar_sri_get /path/to/name.nar.xz name"
 113     restore_path="${scratch_dir}/$2"
 114     xz -d < "$1" | nix-store --restore "${restore_path}"
 115     [[ $? -ne 0 ]] && die "Failed to unpack '$1'"
 116
 117     store_path=$(nix-store --add "${restore_path}")
 118     [[ $? -ne 0 ]] && die "Failed to add '$restore_path' to store"
 119     rm -rf -- "${restore_path}"
 120
 121     nix-hash --to-sri "$(nix-store --query --hash "${store_path}")"
 122 }
 123
 124 # collect passed options
 125
 126 targets=()
 127 commit=no
 128
 129 for arg in "$@"; do
 130     case "$arg" in
 131         --all-targets)
 132             targets+=(
 133                 ${CROSS_TARGETS[@]}
 134                 ${NATIVE_TARGETS[@]}
 135             )
 136             ;;
 137         --targets=*)
 138             # Convert "--targets=a,b,c" to targets=(a b c) bash array.
 139             comma_targets=${arg#--targets=}
 140             targets+=(${comma_targets//,/ })
 141             ;;
 142         --commit)
 143             commit=yes
 144             ;;
 145         *)
 146             usage
 147             ;;
 148     esac
 149 done
 150
 151 for target in "${targets[@]}"; do
 152     # Native and cross jobsets differ a bit. We'll have to pick the
 153     # one based on target name:
 154     if is_native $target; then
 155         jobset=nixpkgs/trunk
 156         job="stdenvBootstrapTools.${target}.build"
 157     elif is_cross $target; then
 158         jobset=nixpkgs/cross-trunk
 159         job="bootstrapTools.${target}.build"
 160     else
 161         die "'$target' is not present in either of 'NATIVE_TARGETS' or 'CROSS_TARGETS'. Please add one."
 162     fi
 163
 164     # 'nixpkgs' prefix where we will write new tarball hashes
 165     case "$target" in
 166         *linux*) nixpkgs_prefix="pkgs/stdenv/linux" ;;
 167         *darwin*) nixpkgs_prefix="pkgs/stdenv/darwin" ;;
 168         *freebsd*) nixpkgs_prefix="pkgs/stdenv/freebsd" ;;
 169         *) die "don't know where to put '$target'" ;;
 170     esac
 171
 172     # We enforce s3 prefix for all targets here. This slightly differs
 173     # from manual uploads targets where names were chosen inconsistently.
 174     s3_prefix="stdenv/$target"
 175
 176     # resolve 'latest' build to the build 'id', construct the link.
 177     latest_build_uri="https://hydra.nixos.org/job/$jobset/$job/latest"
 178     latest_build="$target.latest-build"
 179     info "Fetching latest successful build from '${latest_build_uri}'"
 180     curl -s -H "Content-Type: application/json" -L "$latest_build_uri" > "$latest_build"
 181     [[ $? -ne 0 ]] && die "Failed to fetch latest successful build"
 182     latest_build_id=$(jq '.id' < "$latest_build")
 183     [[ $? -ne 0 ]] && die "Did not find 'id' in latest build"
 184     build_uri="https://hydra.nixos.org/build/${latest_build_id}"
 185
 186     # We pick oldest jobset evaluation and extract the 'nicpkgs' commit.
 187     #
 188     # We use oldest instead of latest to make the result more stable
 189     # across unrelated 'nixpkgs' updates. Ideally two subsequent runs of
 190     # this refresher should produce the same output (provided there are
 191     # no bootstrapTools updates committed between the two runs).
 192     oldest_eval_id=$(jq '.jobsetevals|min' < "$latest_build")
 193     [[ $? -ne 0 ]] && die "Did not find 'jobsetevals' in latest build"
 194     eval_uri="https://hydra.nixos.org/eval/${oldest_eval_id}"
 195     eval_meta="$target.eval-meta"
 196     info "Fetching oldest eval details from '${eval_uri}' (can take a minute)"
 197     curl -s -H "Content-Type: application/json"  -L "${eval_uri}" > "$eval_meta"
 198     [[ $? -ne 0 ]] && die "Failed to fetch eval metadata"
 199     nixpkgs_revision=$(jq --raw-output ".jobsetevalinputs.nixpkgs.revision" < "$eval_meta")
 200     [[ $? -ne 0 ]] && die "Failed to fetch revision"
 201
 202     # Extract the build paths out of the build metadata
 203     drvpath=$(jq --raw-output '.drvpath' < "${latest_build}")
 204     [[ $? -ne 0 ]] && die "Did not find 'drvpath' in latest build"
 205     outpath=$(jq --raw-output '.buildoutputs.out.path' < "${latest_build}")
 206     [[ $? -ne 0 ]] && die "Did not find 'buildoutputs' in latest build"
 207     build_timestamp=$(jq --raw-output '.timestamp' < "${latest_build}")
 208     [[ $? -ne 0 ]] && die "Did not find 'timestamp' in latest build"
 209     build_time=$(TZ=UTC LANG=C date --date="@${build_timestamp}" --rfc-email)
 210     [[ $? -ne 0 ]] && die "Failed to format timestamp"
 211
 212     info "Fetching bootstrap tools to calculate hashes from '${outpath}'"
 213     nix-store --realize "$outpath"
 214     [[ $? -ne 0 ]] && die "Failed to fetch '${outpath}' from hydra"
 215
 216     fnames=()
 217
 218     target_file="${nixpkgs_prefix}/bootstrap-files/${target}.nix"
 219     info "Writing '${target_file}'"
 220     {
 221         # header
 222         cat <<EOF
 223 # Autogenerated by maintainers/scripts/bootstrap-files/refresh-tarballs.bash as:
 224 # $ ./refresh-tarballs.bash --targets=${target}
 225 #
 226 # Metadata:
 227 # - nixpkgs revision: ${nixpkgs_revision}
 228 # - hydra build: ${latest_build_uri}
 229 # - resolved hydra build: ${build_uri}
 230 # - instantiated derivation: ${drvpath}
 231 # - output directory: ${outpath}
 232 # - build time: ${build_time}
 233 {
 234 EOF
 235       for p in "${outpath}/on-server"/*; do
 236           fname=$(basename "$p")
 237           fnames+=("$fname")
 238           case "$fname" in
 239               bootstrap-tools.tar.xz) attr=bootstrapTools ;;
 240               busybox) attr=$fname ;;
 241               unpack.nar.xz) attr=unpack ;;
 242               *) die "Don't know how to map '$fname' to attribute name. Please update me."
 243           esac
 244
 245           executable_arg=
 246           executable_nix=
 247           if [[ -x "$p" ]]; then
 248               executable_arg="--executable"
 249               executable_nix="executable = true;"
 250           fi
 251           unpack_nix=
 252           name_nix=
 253           if [[ $fname = *.nar.xz ]]; then
 254               unpack_nix="unpack = true;"
 255               name_nix="name = \"${fname%.nar.xz}\";"
 256               sri=$(nar_sri_get "$p" "${fname%.nar.xz}")
 257               [[ $? -ne 0 ]] && die "Failed to get hash of '$p'"
 258           else
 259               sha256=$(nix-prefetch-url $executable_arg --name "$fname" "file://$p")
 260               [[ $? -ne 0 ]] && die "Failed to get the hash for '$p'"
 261               sri=$(nix-hash --to-sri "sha256:$sha256")
 262               [[ $? -ne 0 ]] && die "Failed to convert '$sha256' hash to an SRI form"
 263           fi
 264
 265           # individual file entries
 266           cat <<EOF
 267   $attr = import <nix/fetchurl.nix> {
 268     url = "http://tarballs.nixos.org/${s3_prefix}/${nixpkgs_revision}/$fname";
 269     hash = "${sri}";$(
 270     [[ -n ${executable_nix} ]] && printf "\n    %s" "${executable_nix}"
 271     [[ -n ${name_nix} ]]       && printf "\n    %s" "${name_nix}"
 272     [[ -n ${unpack_nix} ]]     && printf "\n    %s" "${unpack_nix}"
 273 )
 274   };
 275 EOF
 276       done
 277       # footer
 278       cat <<EOF
 279 }
 280 EOF
 281     } > "${target_file}"
 282
 283         target_file_commit_msg=${target}.commit_message
 284         cat > "$target_file_commit_msg" <<EOF
 285 ${nixpkgs_prefix}: update ${target} bootstrap-files
 286
 287 sha256sum of files to be uploaded:
 288
 289 $(
 290 echo "$ sha256sum ${outpath}/on-server/*"
 291 sha256sum ${outpath}/on-server/*
 292 )
 293
 294 Suggested commands to upload files to 'tarballs.nixos.org':
 295
 296     $ nix-store --realize ${outpath}
 297     $ aws s3 cp --recursive --acl public-read ${outpath}/on-server/ s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision}
 298     $ aws s3 cp --recursive s3://nixpkgs-tarballs/${s3_prefix}/${nixpkgs_revision} ./
 299     $ sha256sum ${fnames[*]}
 300     $ sha256sum ${outpath}/on-server/*
 301 EOF
 302
 303     cat "$target_file_commit_msg"
 304     if [[ $commit == yes ]]; then
 305         git commit "${target_file}" -F "$target_file_commit_msg"
 306     else
 307         info "DRY RUN: git commit ${target_file} -F $target_file_commit_msg"
 308     fi
 309     rm -- "$target_file_commit_msg"
 310
 311     # delete temp files
 312     rm -- "$latest_build" "$eval_meta"
 313 done