pkgs/development/python-modules/dask/default.nix

   1 {
   2   lib,
   3   stdenv,
   4   buildPythonPackage,
   5   fetchFromGitHub,
   6
   7   # build-system
   8   setuptools,
   9
  10   # dependencies
  11   click,
  12   cloudpickle,
  13   fsspec,
  14   importlib-metadata,
  15   packaging,
  16   partd,
  17   pyyaml,
  18   toolz,
  19
  20   # optional-dependencies
  21   numpy,
  22   pyarrow,
  23   lz4,
  24   pandas,
  25   distributed,
  26   bokeh,
  27   jinja2,
  28
  29   # tests
  30   arrow-cpp,
  31   dask-expr,
  32   hypothesis,
  33   pytest-asyncio,
  34   pytest-rerunfailures,
  35   pytest-xdist,
  36   pytestCheckHook,
  37 }:
  38
  39 let
  40   self = buildPythonPackage rec {
  41     pname = "dask";
  42     version = "2024.12.1";
  43     pyproject = true;
  44
  45     src = fetchFromGitHub {
  46       owner = "dask";
  47       repo = "dask";
  48       tag = version;
  49       hash = "sha256-QqvdldAHW2UYt1NXfk3Aa+oe97e+OpRbF8d6eKV3OJ4=";
  50     };
  51
  52     build-system = [ setuptools ];
  53
  54     dependencies = [
  55       click
  56       cloudpickle
  57       fsspec
  58       packaging
  59       partd
  60       pyyaml
  61       importlib-metadata
  62       toolz
  63     ];
  64
  65     optional-dependencies = lib.fix (self: {
  66       array = [ numpy ];
  67       complete =
  68         [
  69           pyarrow
  70           lz4
  71         ]
  72         ++ self.array
  73         ++ self.dataframe
  74         ++ self.distributed
  75         ++ self.diagnostics;
  76       dataframe = [
  77         # dask-expr -> circular dependency with dask-expr
  78         numpy
  79         pandas
  80       ];
  81       distributed = [ distributed ];
  82       diagnostics = [
  83         bokeh
  84         jinja2
  85       ];
  86     });
  87
  88     nativeCheckInputs =
  89       [
  90         dask-expr
  91         pytestCheckHook
  92         pytest-rerunfailures
  93         pytest-xdist
  94         # from panda[test]
  95         hypothesis
  96         pytest-asyncio
  97       ]
  98       ++ self.optional-dependencies.array
  99       ++ self.optional-dependencies.dataframe
 100       ++ lib.optionals (!arrow-cpp.meta.broken) [
 101         # support is sparse on aarch64
 102         pyarrow
 103       ];
 104
 105     dontUseSetuptoolsCheck = true;
 106
 107     postPatch = ''
 108       # versioneer hack to set version of GitHub package
 109       echo "def get_versions(): return {'dirty': False, 'error': None, 'full-revisionid': None, 'version': '${version}'}" > dask/_version.py
 110
 111       substituteInPlace setup.py \
 112         --replace-fail "import versioneer" "" \
 113         --replace-fail "version=versioneer.get_version()," "version='${version}'," \
 114         --replace-fail "cmdclass=versioneer.get_cmdclass()," ""
 115
 116       substituteInPlace pyproject.toml \
 117         --replace-fail ', "versioneer[toml]==0.29"' "" \
 118         --replace-fail " --durations=10" "" \
 119         --replace-fail " --cov-config=pyproject.toml" "" \
 120         --replace-fail "\"-v" "\" "
 121     '';
 122
 123     pytestFlagsArray = [
 124       # Rerun failed tests up to three times
 125       "--reruns 3"
 126       # Don't run tests that require network access
 127       "-m 'not network'"
 128     ];
 129
 130     disabledTests =
 131       lib.optionals stdenv.hostPlatform.isDarwin [
 132         # Test requires features of python3Packages.psutil that are
 133         # blocked in sandboxed-builds
 134         "test_auto_blocksize_csv"
 135         # AttributeError: 'str' object has no attribute 'decode'
 136         "test_read_dir_nometa"
 137       ]
 138       ++ lib.optionals (stdenv.hostPlatform.isDarwin && stdenv.hostPlatform.isAarch64) [
 139         # concurrent.futures.process.BrokenProcessPool: A process in the process pool terminated abpruptly...
 140         "test_foldby_tree_reduction"
 141         "test_to_bag"
 142       ]
 143       ++ [
 144         # https://github.com/dask/dask/issues/10347#issuecomment-1589683941
 145         "test_concat_categorical"
 146         # AttributeError: 'ArrowStringArray' object has no attribute 'tobytes'. Did you mean: 'nbytes'?
 147         "test_dot"
 148         "test_dot_nan"
 149         "test_merge_column_with_nulls"
 150         # FileNotFoundError: [Errno 2] No such file or directory: '/build/tmp301jryv_/createme/0.part'
 151         "test_to_csv_nodir"
 152         "test_to_json_results"
 153         # FutureWarning: Those tests should be working fine when pandas will have been upgraded to 2.1.1
 154         "test_apply"
 155         "test_apply_infer_columns"
 156       ];
 157
 158     __darwinAllowLocalNetworking = true;
 159
 160     pythonImportsCheck = [
 161       "dask"
 162       "dask.bag"
 163       "dask.bytes"
 164       "dask.diagnostics"
 165     ];
 166
 167     doCheck = false;
 168
 169     # Enable tests via passthru to avoid cyclic dependency with dask-expr.
 170     passthru.tests = {
 171       check = self.overridePythonAttrs (old: {
 172         doCheck = true;
 173         pythonImportsCheck = [
 174           # Requires the `dask.optional-dependencies.array` that are only in `nativeCheckInputs`
 175           "dask.array"
 176           # Requires the `dask.optional-dependencies.dataframe` that are only in `nativeCheckInputs`
 177           "dask.dataframe"
 178           "dask.dataframe.io"
 179           "dask.dataframe.tseries"
 180         ] ++ old.pythonImportsCheck;
 181       });
 182     };
 183
 184     meta = {
 185       description = "Minimal task scheduling abstraction";
 186       mainProgram = "dask";
 187       homepage = "https://dask.org/";
 188       changelog = "https://docs.dask.org/en/latest/changelog.html";
 189       license = lib.licenses.bsd3;
 190       maintainers = with lib.maintainers; [ GaetanLepage ];
 191     };
 192   };
 193 in
 194 self