lib/fileset/benchmark.sh

   1 #!/usr/bin/env nix-shell
   2 #!nix-shell -i bash -p sta jq bc nix -I nixpkgs=../..
   3 # shellcheck disable=SC2016
   4
   5 # Benchmarks lib.fileset
   6 # Run:
   7 # [nixpkgs]$ lib/fileset/benchmark.sh HEAD
   8
   9 set -euo pipefail
  10 shopt -s inherit_errexit dotglob
  11
  12 if (( $# == 0 )); then
  13     echo "Usage: $0 HEAD"
  14     echo "Benchmarks the current tree against the HEAD commit. Any git ref will work."
  15     exit 1
  16 fi
  17 compareTo=$1
  18
  19 SCRIPT_FILE=$(readlink -f "${BASH_SOURCE[0]}")
  20 SCRIPT_DIR=$(dirname "$SCRIPT_FILE")
  21
  22 nixpkgs=$(cd "$SCRIPT_DIR/../.."; pwd)
  23
  24 tmp="$(mktemp -d)"
  25 clean_up() {
  26     rm -rf "$tmp"
  27 }
  28 trap clean_up EXIT SIGINT SIGTERM
  29 work="$tmp/work"
  30 mkdir "$work"
  31 cd "$work"
  32
  33 declare -a stats=(
  34     ".envs.elements"
  35     ".envs.number"
  36     ".gc.totalBytes"
  37     ".list.concats"
  38     ".list.elements"
  39     ".nrFunctionCalls"
  40     ".nrLookups"
  41     ".nrOpUpdates"
  42     ".nrPrimOpCalls"
  43     ".nrThunks"
  44     ".sets.elements"
  45     ".sets.number"
  46     ".symbols.number"
  47     ".values.number"
  48 )
  49
  50 runs=10
  51
  52 run() {
  53     # Empty the file
  54     : > cpuTimes
  55
  56     for i in $(seq 0 "$runs"); do
  57         NIX_PATH=nixpkgs=$1 NIX_SHOW_STATS=1 NIX_SHOW_STATS_PATH=$tmp/stats.json \
  58             nix-instantiate --eval --strict --show-trace >/dev/null \
  59             --expr 'with import <nixpkgs/lib>; with fileset; '"$2"
  60
  61         # Only measure the time after the first run, one is warmup
  62         if (( i > 0 )); then
  63             jq '.cpuTime' "$tmp/stats.json" >> cpuTimes
  64         fi
  65     done
  66
  67     # Compute mean and standard deviation
  68     read -r mean sd < <(sta --mean --sd --brief <cpuTimes)
  69
  70     jq --argjson mean "$mean" --argjson sd "$sd" \
  71         '.cpuTimeMean = $mean | .cpuTimeSd = $sd' \
  72         "$tmp/stats.json"
  73 }
  74
  75 bench() {
  76     echo "Benchmarking expression $1" >&2
  77     #echo "Running benchmark on index" >&2
  78     run "$nixpkgs" "$1" > "$tmp/new.json"
  79     (
  80         #echo "Checking out $compareTo" >&2
  81         git -C "$nixpkgs" worktree add --quiet "$tmp/worktree" "$compareTo"
  82         trap 'git -C "$nixpkgs" worktree remove "$tmp/worktree"' EXIT
  83         #echo "Running benchmark on $compareTo" >&2
  84         run "$tmp/worktree" "$1" > "$tmp/old.json"
  85     )
  86
  87     read -r oldMean oldSd newMean newSd percentageMean percentageSd < \
  88         <(jq -rn --slurpfile old "$tmp/old.json" --slurpfile new "$tmp/new.json" \
  89         ' $old[0].cpuTimeMean as $om
  90         | $old[0].cpuTimeSd as $os
  91         | $new[0].cpuTimeMean as $nm
  92         | $new[0].cpuTimeSd as $ns
  93         | (100 / $om * $nm) as $pm
  94         # Copied from https://github.com/sharkdp/hyperfine/blob/b38d550b89b1dab85139eada01c91a60798db9cc/src/benchmark/relative_speed.rs#L46-L53
  95         | ($pm * pow(pow($ns / $nm; 2) + pow($os / $om; 2); 0.5)) as $ps
  96         | [ $om, $os, $nm, $ns, $pm, $ps ]
  97         | @sh')
  98
  99     echo -e "Mean CPU time $newMean (σ = $newSd) for $runs runs is \e[0;33m$percentageMean% (σ = $percentageSd%)\e[0m of the old value $oldMean (σ = $oldSd)" >&2
 100
 101     different=0
 102     for stat in "${stats[@]}"; do
 103         oldValue=$(jq "$stat" "$tmp/old.json")
 104         newValue=$(jq "$stat" "$tmp/new.json")
 105         if (( oldValue != newValue )); then
 106             percent=$(bc <<< "scale=100; result = 100/$oldValue*$newValue; scale=4; result / 1")
 107             if (( oldValue < newValue )); then
 108                 echo -e "Statistic $stat ($newValue) is \e[0;31m$percent% (+$(( newValue - oldValue )))\e[0m of the old value $oldValue" >&2
 109             else
 110                 echo -e "Statistic $stat ($newValue) is \e[0;32m$percent% (-$(( oldValue - newValue )))\e[0m of the old value $oldValue" >&2
 111             fi
 112             (( different++ )) || true
 113         fi
 114     done
 115     echo "$different stats differ between the current tree and $compareTo"
 116     echo ""
 117 }
 118
 119 # Create a fairly populated tree
 120 touch f{0..5}
 121 mkdir d{0..5}
 122 mkdir e{0..5}
 123 touch d{0..5}/f{0..5}
 124 mkdir -p d{0..5}/d{0..5}
 125 mkdir -p e{0..5}/e{0..5}
 126 touch d{0..5}/d{0..5}/f{0..5}
 127 mkdir -p d{0..5}/d{0..5}/d{0..5}
 128 mkdir -p e{0..5}/e{0..5}/e{0..5}
 129 touch d{0..5}/d{0..5}/d{0..5}/f{0..5}
 130 mkdir -p d{0..5}/d{0..5}/d{0..5}/d{0..5}
 131 mkdir -p e{0..5}/e{0..5}/e{0..5}/e{0..5}
 132 touch d{0..5}/d{0..5}/d{0..5}/d{0..5}/f{0..5}
 133
 134 bench 'toSource { root = ./.; fileset = ./.; }'
 135
 136 rm -rf -- *
 137
 138 touch {0..1000}
 139 bench 'toSource { root = ./.; fileset = unions (mapAttrsToList (name: value: ./. + "/${name}") (builtins.readDir ./.)); }'
 140 rm -rf -- *