maintainers/scripts/rebuild-amount.sh

   1 #!/usr/bin/env bash
   2 set -e
   3
   4 # --print: avoid dependency on environment
   5 optPrint=
   6 if [ "$1" == "--print" ]; then
   7     optPrint=true
   8     shift
   9 fi
  10
  11 if [ "$#" != 1 ] && [ "$#" != 2 ]; then
  12     cat <<EOF
  13     Usage: $0 [--print] from-commit-spec [to-commit-spec]
  14         You need to be in a git-controlled nixpkgs tree.
  15         The current state of the tree will be used if the second commit is missing.
  16
  17         Examples:
  18           effect of latest commit:
  19               $ $0 HEAD^
  20               $ $0 --print HEAD^
  21           effect of the whole patch series for 'staging' branch:
  22               $ $0 origin/staging staging
  23 EOF
  24     exit 1
  25 fi
  26
  27 # A slightly hacky way to get the config.
  28 parallel="$(echo 'config.rebuild-amount.parallel or false' | nix-repl . 2>/dev/null \
  29             | grep -v '^\(nix-repl.*\)\?$' | tail -n 1 || true)"
  30
  31 echo "Estimating rebuild amount by counting changed Hydra jobs (parallel=${parallel:-unset})."
  32
  33 toRemove=()
  34
  35 cleanup() {
  36     rm -rf "${toRemove[@]}"
  37 }
  38 trap cleanup EXIT
  39
  40 MKTEMP='mktemp --tmpdir nix-rebuild-amount-XXXXXXXX'
  41
  42 nixexpr() {
  43     cat <<EONIX
  44         let
  45           lib = import $1/lib;
  46           hydraJobs = import $1/pkgs/top-level/release.nix
  47             # Compromise: accuracy vs. resources needed for evaluation.
  48             { supportedSystems = cfg.systems or [ "x86_64-linux" "x86_64-darwin" ]; };
  49           cfg = (import $1 {}).config.rebuild-amount or {};
  50
  51           recurseIntoAttrs = attrs: attrs // { recurseForDerivations = true; };
  52
  53           # hydraJobs leaves recurseForDerivations as empty attrmaps;
  54           # that would break nix-env and we also need to recurse everywhere.
  55           tweak = lib.mapAttrs
  56             (name: val:
  57               if name == "recurseForDerivations" then true
  58               else if lib.isAttrs val && val.type or null != "derivation"
  59                       then recurseIntoAttrs (tweak val)
  60               else val
  61             );
  62
  63           # Some of these contain explicit references to platform(s) we want to avoid;
  64           # some even (transitively) depend on ~/.nixpkgs/config.nix (!)
  65           blacklist = [
  66             "tarball" "metrics" "manual"
  67             "darwin-tested" "unstable" "stdenvBootstrapTools"
  68             "moduleSystem" "lib-tests" # these just confuse the output
  69           ];
  70
  71         in
  72           tweak (builtins.removeAttrs hydraJobs blacklist)
  73 EONIX
  74 }
  75
  76 # Output packages in tree $2 that weren't in $1.
  77 # Changing the output hash or name is taken as a change.
  78 # Extra nix-env parameters can be in $3
  79 newPkgs() {
  80     # We use files instead of pipes, as running multiple nix-env processes
  81     # could eat too much memory for a standard 4GiB machine.
  82     local -a list
  83     for i in 1 2; do
  84         local l
  85         l="$($MKTEMP)"
  86         list[$i]="$l"
  87         toRemove+=("$l")
  88
  89         local expr
  90         expr="$($MKTEMP)"
  91         toRemove+=("$expr")
  92         nixexpr "${!i}" > "$expr"
  93
  94         nix-env -f "$expr" -qaP --no-name --out-path --show-trace $3 \
  95             | sort > "${list[$i]}" &
  96
  97         if [ "$parallel" != "true" ]; then
  98             wait
  99         fi
 100     done
 101
 102     wait
 103     comm -13 "${list[@]}"
 104 }
 105
 106 # Prepare nixpkgs trees.
 107 declare -a tree
 108 for i in 1 2; do
 109     if [ -n "${!i}" ]; then # use the given commit
 110         dir="$($MKTEMP -d)"
 111         tree[$i]="$dir"
 112         toRemove+=("$dir")
 113
 114         git clone --shared --no-checkout --quiet . "${tree[$i]}"
 115         (cd "${tree[$i]}" && git checkout --quiet "${!i}")
 116     else #use the current tree
 117         tree[$i]="$(pwd)"
 118     fi
 119 done
 120
 121 newlist="$($MKTEMP)"
 122 toRemove+=("$newlist")
 123 # Notes:
 124 #    - the evaluation is done on x86_64-linux, like on Hydra.
 125 #    - using $newlist file so that newPkgs() isn't in a sub-shell (because of toRemove)
 126 newPkgs "${tree[1]}" "${tree[2]}" '--argstr system "x86_64-linux"' > "$newlist"
 127
 128 # Hacky: keep only the last word of each attribute path and sort.
 129 sed -n 's/\([^. ]*\.\)*\([^. ]*\) .*$/\2/p' < "$newlist" \
 130     | sort | uniq -c
 131
 132 if [ -n "$optPrint" ]; then
 133     echo
 134     cat "$newlist"
 135 fi