pkgs/development/python-modules/semgrep/update.sh

   1 #!/usr/bin/env nix-shell
   2 #!nix-shell -i bash -p curl gnused jq nix-prefetch
   3
   4 set -euxo pipefail
   5
   6 # provide a github token so you don't get rate limited
   7 # if you use gh cli you can use:
   8 #     `export GITHUB_TOKEN="$(cat ~/.config/gh/config.yml | yq '.hosts."github.com".oauth_token' -r)"`
   9 # or just set your token by hand:
  10 #     `read -s -p "Enter your token: " GITHUB_TOKEN; export GITHUB_TOKEN`
  11 #     (we use read so it doesn't show in our shell history and in secret mode so the token you paste isn't visible)
  12 if [ -z "${GITHUB_TOKEN:-}" ]; then
  13     echo "no GITHUB_TOKEN provided - you could meet API request limiting" >&2
  14 fi
  15
  16 ROOT="$(dirname "$(readlink -f "$0")")"
  17 NIXPKGS_ROOT="$ROOT/../../../.."
  18
  19 COMMON_FILE="$ROOT/common.nix"
  20
  21 instantiateClean() {
  22     nix-instantiate -A "$1" --eval --strict | cut -d\" -f2
  23 }
  24
  25 # get latest version
  26 NEW_VERSION=$(
  27   curl -s -L -H \
  28     "Accept: application/vnd.github.v3+json" \
  29     ${GITHUB_TOKEN:+ -H "Authorization: bearer $GITHUB_TOKEN"} \
  30     https://api.github.com/repos/semgrep/semgrep/releases/latest \
  31   | jq -r '.tag_name'
  32 )
  33 # trim v prefix
  34 NEW_VERSION="${NEW_VERSION:1}"
  35 OLD_VERSION="$(instantiateClean semgrep.passthru.common.version)"
  36
  37 if [[ "$OLD_VERSION" == "$NEW_VERSION" ]]; then
  38     echo "Already up to date"
  39     exit
  40 fi
  41
  42 replace() {
  43     sed -i "s@$1@$2@g" "$3"
  44 }
  45
  46 fetchgithub() {
  47     set +eo pipefail
  48     nix-build -A "$1" 2>&1 >/dev/null | grep "got:" | cut -d':' -f2 | sed 's| ||g'
  49     set -eo pipefail
  50 }
  51
  52 fetch_arch() {
  53   VERSION=$1
  54   PLATFORM=$2
  55   nix-prefetch "{ fetchPypi }:
  56 fetchPypi rec {
  57   pname = \"semgrep\";
  58   version = \"$VERSION\";
  59   format = \"wheel\";
  60   dist = python;
  61   python = \"cp38.cp39.cp310.cp311.py37.py38.py39.py310.py311\";
  62   platform = \"$PLATFORM\";
  63 }
  64 "
  65 }
  66
  67 replace "$OLD_VERSION" "$NEW_VERSION" "$COMMON_FILE"
  68
  69 echo "Updating src"
  70
  71 OLD_HASH="$(instantiateClean semgrep.passthru.common.srcHash)"
  72 echo "Old hash $OLD_HASH"
  73 TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
  74 replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE"
  75 NEW_HASH="$(fetchgithub semgrep.src)"
  76 echo "New hash $NEW_HASH"
  77 replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE"
  78
  79 echo "Updated src"
  80
  81
  82 update_core_platform() {
  83     SYSTEM=$1
  84     echo "Updating core src $SYSTEM"
  85
  86     PLATFORM="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.platform")"
  87
  88     OLD_HASH="$(instantiateClean "semgrep.passthru.common.core.$SYSTEM.hash")"
  89     echo "Old core hash $OLD_HASH"
  90     NEW_HASH="$(fetch_arch "$NEW_VERSION" "$PLATFORM")"
  91     echo "New core hash $NEW_HASH"
  92     replace "$OLD_HASH" "$NEW_HASH" "$COMMON_FILE"
  93
  94     echo "Updated core src $SYSTEM"
  95 }
  96
  97 update_core_platform "x86_64-linux"
  98 update_core_platform "aarch64-linux"
  99 update_core_platform "x86_64-darwin"
 100 update_core_platform "aarch64-darwin"
 101
 102 OLD_PWD=$PWD
 103 TMPDIR="$(mktemp -d)"
 104 # shallow clone to check submodule commits, don't actually need the submodules
 105 git clone https://github.com/semgrep/semgrep "$TMPDIR/semgrep" --depth 1 --branch "v$NEW_VERSION"
 106
 107 get_submodule_commit() {
 108     OLD_PWD=$PWD
 109     (
 110         cd "$TMPDIR/semgrep"
 111         git ls-tree --object-only HEAD "$1"
 112         cd "$OLD_PWD"
 113     )
 114 }
 115
 116 # loop through submodules
 117 nix-instantiate -E "with import $NIXPKGS_ROOT {}; builtins.attrNames semgrep.passthru.common.submodules" --eval --strict --json \
 118 | jq '.[]' -r \
 119 | while read -r SUBMODULE; do
 120     echo "Updating $SUBMODULE"
 121     OLD_REV=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".rev)
 122     echo "Old commit $OLD_REV"
 123     OLD_HASH=$(instantiateClean semgrep.passthru.common.submodules."$SUBMODULE".hash)
 124     echo "Old hash $OLD_HASH"
 125
 126     NEW_REV=$(get_submodule_commit "$SUBMODULE")
 127     echo "New commit $NEW_REV"
 128
 129     if [[ "$OLD_REV" == "$NEW_REV" ]]; then
 130       echo "$SUBMODULE already up to date"
 131       continue
 132     fi
 133
 134     TMP_HASH="sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="
 135     replace "$OLD_REV" "$NEW_REV" "$COMMON_FILE"
 136     replace "$OLD_HASH" "$TMP_HASH" "$COMMON_FILE"
 137     NEW_HASH="$(fetchgithub semgrep.passthru.submodulesSubset."$SUBMODULE")"
 138     echo "New hash $NEW_HASH"
 139     replace "$TMP_HASH" "$NEW_HASH" "$COMMON_FILE"
 140
 141     echo "Updated $SUBMODULE"
 142 done
 143
 144 rm -rf "$TMPDIR"
 145
 146 echo "Finished"
 147