pkgs/development/python-modules/spacy/models.nix

   1 {
   2   lib,
   3   buildPythonPackage,
   4   fetchurl,
   5   protobuf,
   6   pymorphy3,
   7   pymorphy3-dicts-uk,
   8   sentencepiece,
   9   setuptools,
  10   spacy,
  11   spacy-pkuseg,
  12   spacy-transformers,
  13   writeScript,
  14   stdenv,
  15   jq,
  16   nix,
  17   moreutils,
  18 }:
  19 let
  20   buildModelPackage =
  21     {
  22       pname,
  23       version,
  24       sha256,
  25       license,
  26     }:
  27
  28     let
  29       lang = builtins.substring 0 2 pname;
  30       requires-protobuf = pname == "fr_dep_news_trf" || pname == "uk_core_news_trf";
  31     in
  32     buildPythonPackage {
  33       inherit pname version;
  34       pyproject = true;
  35
  36       src = fetchurl {
  37         url = "https://github.com/explosion/spacy-models/releases/download/${pname}-${version}/${pname}-${version}.tar.gz";
  38         inherit sha256;
  39       };
  40
  41       propagatedBuildInputs =
  42         [ spacy ]
  43         ++ lib.optionals (lib.hasSuffix "_trf" pname) [ spacy-transformers ]
  44         ++ lib.optionals (lang == "ru") [ pymorphy3 ]
  45         ++ lib.optionals (lang == "uk") [
  46           pymorphy3
  47           pymorphy3-dicts-uk
  48         ]
  49         ++ lib.optionals (lang == "zh") [ spacy-pkuseg ]
  50         ++ lib.optionals (pname == "fr_dep_news_trf") [ sentencepiece ];
  51
  52       postPatch = lib.optionalString requires-protobuf ''
  53         substituteInPlace meta.json \
  54           --replace "protobuf<3.21.0" "protobuf"
  55       '';
  56
  57       nativeBuildInputs = [ setuptools ] ++ lib.optionals requires-protobuf [ protobuf ];
  58
  59       pythonImportsCheck = [ pname ];
  60
  61       passthru.updateScript = writeScript "update-spacy-models" ''
  62         #!${stdenv.shell}
  63         set -eou pipefail
  64         PATH=${
  65           lib.makeBinPath [
  66             jq
  67             nix
  68             moreutils
  69           ]
  70         }
  71
  72         IFS=. read -r major minor patch <<<"${spacy.version}"
  73         spacyVersion="$(echo "$major.$minor.0")"
  74
  75         pushd pkgs/development/python-modules/spacy/ || exit
  76
  77         jq -r '.[] | .pname' models.json | while IFS= read -r pname; do
  78           if [ "$(jq --arg pname "$pname" -r '.[] | select(.pname == $pname) | .version' models.json)" == "$spacyVersion" ]; then
  79             continue
  80           fi
  81
  82           newHash="$(nix-prefetch-url "https://github.com/explosion/spacy-models/releases/download/$pname-$spacyVersion/$pname-$spacyVersion.tar.gz")"
  83           jq --arg newHash "$newHash" --arg pname "$pname" --arg spacyVersion "$spacyVersion" \
  84            '[(.[] | select(.pname != $pname)), (.[] | select(.pname == $pname) | .sha256 = $newHash | .version = $spacyVersion)] | sort_by(.pname)' \
  85            models.json | sponge models.json
  86         done
  87
  88         popd || exit
  89       '';
  90
  91       meta = {
  92         description = "Models for the spaCy NLP library";
  93         homepage = "https://github.com/explosion/spacy-models";
  94         license = lib.licenses.${license};
  95       };
  96     };
  97
  98   makeModelSet =
  99     models: lib.listToAttrs (map (m: lib.nameValuePair m.pname (buildModelPackage m)) models);
 100 in
 101 makeModelSet (lib.importJSON ./models.json)