pkgs/development/python-modules/transformers/default.nix

   1 {
   2   lib,
   3   stdenv,
   4   buildPythonPackage,
   5   fetchFromGitHub,
   6
   7   # build-system
   8   setuptools,
   9
  10   # dependencies
  11   filelock,
  12   huggingface-hub,
  13   numpy,
  14   protobuf,
  15   packaging,
  16   pyyaml,
  17   regex,
  18   requests,
  19   tokenizers,
  20   safetensors,
  21   tqdm,
  22
  23   # optional-dependencies
  24   diffusers,
  25   scikit-learn,
  26   tensorflow,
  27   onnxconverter-common,
  28   opencv4,
  29   tf2onnx,
  30   torch,
  31   accelerate,
  32   faiss,
  33   datasets,
  34   jax,
  35   jaxlib,
  36   flax,
  37   optax,
  38   ftfy,
  39   onnxruntime,
  40   onnxruntime-tools,
  41   cookiecutter,
  42   sagemaker,
  43   fairscale,
  44   optuna,
  45   ray,
  46   pydantic,
  47   uvicorn,
  48   fastapi,
  49   starlette,
  50   librosa,
  51   phonemizer,
  52   torchaudio,
  53   pillow,
  54   timm,
  55   torchvision,
  56   av,
  57   sentencepiece,
  58 }:
  59
  60 buildPythonPackage rec {
  61   pname = "transformers";
  62   version = "4.47.0";
  63   pyproject = true;
  64
  65   src = fetchFromGitHub {
  66     owner = "huggingface";
  67     repo = "transformers";
  68     tag = "v${version}";
  69     hash = "sha256-TQQ+w+EH/KWLE7iaaAHGxfE74hCiLXcqlIr1TIBFGvo=";
  70   };
  71
  72   # torch.distributed is not available on darwin
  73   # Fix submitted upstream in https://github.com/huggingface/transformers/pull/35133
  74   postPatch = lib.optionalString stdenv.hostPlatform.isDarwin ''
  75     substituteInPlace src/transformers/pytorch_utils.py \
  76       --replace-fail \
  77         'if is_torch_greater_or_equal("2.5"):' \
  78         'if False:'
  79   '';
  80
  81   build-system = [ setuptools ];
  82
  83   dependencies = [
  84     filelock
  85     huggingface-hub
  86     numpy
  87     packaging
  88     pyyaml
  89     regex
  90     requests
  91     tokenizers
  92     safetensors
  93     tqdm
  94   ];
  95
  96   optional-dependencies =
  97     let
  98       audio = [
  99         librosa
 100         # pyctcdecode
 101         phonemizer
 102         # kenlm
 103       ];
 104       vision = [ pillow ];
 105     in
 106     {
 107       agents = [
 108         diffusers
 109         accelerate
 110         datasets
 111         torch
 112         sentencepiece
 113         opencv4
 114         pillow
 115       ];
 116       ja = [
 117         # fugashi
 118         # ipadic
 119         # rhoknp
 120         # sudachidict_core
 121         # sudachipy
 122         # unidic
 123         # unidic_lite
 124       ];
 125       sklearn = [ scikit-learn ];
 126       tf = [
 127         tensorflow
 128         onnxconverter-common
 129         tf2onnx
 130         # tensorflow-text
 131         # keras-nlp
 132       ];
 133       torch = [
 134         torch
 135         accelerate
 136       ];
 137       retrieval = [
 138         faiss
 139         datasets
 140       ];
 141       flax = [
 142         jax
 143         jaxlib
 144         flax
 145         optax
 146       ];
 147       tokenizers = [ tokenizers ];
 148       ftfy = [ ftfy ];
 149       onnxruntime = [
 150         onnxruntime
 151         onnxruntime-tools
 152       ];
 153       onnx = [
 154         onnxconverter-common
 155         tf2onnx
 156         onnxruntime
 157         onnxruntime-tools
 158       ];
 159       modelcreation = [ cookiecutter ];
 160       sagemaker = [ sagemaker ];
 161       deepspeed = [
 162         # deepspeed
 163         accelerate
 164       ];
 165       fairscale = [ fairscale ];
 166       optuna = [ optuna ];
 167       ray = [ ray ] ++ ray.optional-dependencies.tune;
 168       # sigopt = [ sigopt ];
 169       # integrations = ray ++ optuna ++ sigopt;
 170       serving = [
 171         pydantic
 172         uvicorn
 173         fastapi
 174         starlette
 175       ];
 176       audio = audio;
 177       speech = [ torchaudio ] ++ audio;
 178       torch-speech = [ torchaudio ] ++ audio;
 179       tf-speech = audio;
 180       flax-speech = audio;
 181       timm = [ timm ];
 182       torch-vision = [ torchvision ] ++ vision;
 183       # natten = [ natten ];
 184       # codecarbon = [ codecarbon ];
 185       video = [
 186         av
 187       ];
 188       sentencepiece = [
 189         sentencepiece
 190         protobuf
 191       ];
 192     };
 193
 194   # Many tests require internet access.
 195   doCheck = false;
 196
 197   pythonImportsCheck = [ "transformers" ];
 198
 199   meta = {
 200     homepage = "https://github.com/huggingface/transformers";
 201     description = "Natural Language Processing for TensorFlow 2.0 and PyTorch";
 202     mainProgram = "transformers-cli";
 203     changelog = "https://github.com/huggingface/transformers/releases/tag/v${version}";
 204     license = lib.licenses.asl20;
 205     platforms = lib.platforms.unix;
 206     maintainers = with lib.maintainers; [
 207       pashashocky
 208       happysalada
 209     ];
 210   };
 211 }