pkgs/development/python-modules/ibis-framework/default.nix

   1 {
   2   lib,
   3   buildPythonPackage,
   4   fetchFromGitHub,
   5   pythonOlder,
   6   pytestCheckHook,
   7   atpublic,
   8   black,
   9   clickhouse-connect,
  10   datafusion,
  11   db-dtypes,
  12   duckdb,
  13   fetchpatch,
  14   filelock,
  15   geopandas,
  16   google-cloud-bigquery,
  17   google-cloud-bigquery-storage,
  18   graphviz,
  19   hypothesis,
  20   numpy,
  21   oracledb,
  22   packaging,
  23   pandas,
  24   parsy,
  25   pins,
  26   poetry-core,
  27   poetry-dynamic-versioning,
  28   polars,
  29   psycopg2,
  30   pyarrow,
  31   pyarrow-hotfix,
  32   pydata-google-auth,
  33   pydruid,
  34   pymysql,
  35   pyodbc,
  36   pyspark,
  37   pytest-benchmark,
  38   pytest-httpserver,
  39   pytest-mock,
  40   pytest-randomly,
  41   pytest-snapshot,
  42   pytest-timeout,
  43   pytest-xdist,
  44   python-dateutil,
  45   pytz,
  46   regex,
  47   rich,
  48   shapely,
  49   snowflake-connector-python,
  50   sqlglot,
  51   sqlite,
  52   toolz,
  53   trino-python-client,
  54   typing-extensions,
  55 }:
  56 let
  57   testBackends = [
  58     "duckdb"
  59     "sqlite"
  60   ];
  61
  62   ibisTestingData = fetchFromGitHub {
  63     name = "ibis-testing-data";
  64     owner = "ibis-project";
  65     repo = "testing-data";
  66     # https://github.com/ibis-project/ibis/blob/9.5.0/nix/overlay.nix#L20-L26
  67     rev = "b26bd40cf29004372319df620c4bbe41420bb6f8";
  68     hash = "sha256-1fenQNQB+Q0pbb0cbK2S/UIwZDE4PXXG15MH3aVbyLU=";
  69   };
  70 in
  71
  72 buildPythonPackage rec {
  73   pname = "ibis-framework";
  74   version = "9.5.0";
  75   pyproject = true;
  76
  77   disabled = pythonOlder "3.10";
  78
  79   src = fetchFromGitHub {
  80     name = "ibis-source";
  81     repo = "ibis";
  82     owner = "ibis-project";
  83     rev = "refs/tags/${version}";
  84     hash = "sha256-6ebw/E3jZFMHKqC5ZY//2Ke0NrklyoGp5JGKBfDxy40=";
  85   };
  86
  87   patches = [
  88     # remove after the 10.0 release
  89     (fetchpatch {
  90       name = "ibis-framework-duckdb-1.1.1.patch";
  91       url = "https://github.com/ibis-project/ibis/commit/a54eceabac1d6592e9f6ab0ca7749e37a748c2ad.patch";
  92       hash = "sha256-j5BPYVqnEF9GQV5N3/VhFUCdsEwAIOQC0KfZ5LNBSRg=";
  93     })
  94
  95     # remove after the 10.0 release
  96     (fetchpatch {
  97       name = "ibis-framework-arrow-18.patch";
  98       url = "https://github.com/ibis-project/ibis/commit/5dc549b22c2eca29a11a31fb29deef7c1466a204.patch";
  99       hash = "sha256-4i/g2uixdlkbE6x659wzZJ91FZpzwOVkF6ZeXkiCP3I=";
 100       excludes = [
 101         "poetry.lock"
 102         "requirements-dev.txt"
 103       ];
 104     })
 105   ];
 106
 107   build-system = [
 108     poetry-core
 109     poetry-dynamic-versioning
 110   ];
 111
 112   dontBypassPoetryDynamicVersioning = true;
 113   env.POETRY_DYNAMIC_VERSIONING_BYPASS = lib.head (lib.strings.splitString "-" version);
 114
 115   dependencies = [
 116     atpublic
 117     parsy
 118     python-dateutil
 119     pytz
 120     sqlglot
 121     toolz
 122     typing-extensions
 123   ];
 124
 125   nativeCheckInputs = [
 126     pytestCheckHook
 127     black
 128     filelock
 129     hypothesis
 130     pytest-benchmark
 131     pytest-httpserver
 132     pytest-mock
 133     pytest-randomly
 134     pytest-snapshot
 135     pytest-timeout
 136     # this dependency is still needed due to use of strict markers and
 137     # `pytest.mark.xdist_group` in the ibis codebase
 138     pytest-xdist
 139   ] ++ lib.concatMap (name: optional-dependencies.${name}) testBackends;
 140
 141   pytestFlagsArray = [
 142     "-m"
 143     "'${lib.concatStringsSep " or " testBackends} or core'"
 144   ];
 145
 146   disabledTests = [
 147     # tries to download duckdb extensions
 148     "test_attach_sqlite"
 149     "test_connect_extensions"
 150     "test_load_extension"
 151     "test_read_sqlite"
 152     "test_register_sqlite"
 153     # requires network connection
 154     "test_s3_403_fallback"
 155     "test_hugging_face"
 156     # requires pytest 8.2+
 157     "test_roundtrip_delta"
 158   ];
 159
 160   # patch out tests that check formatting with black
 161   postPatch = ''
 162     find ibis/tests -type f -name '*.py' -exec sed -i \
 163       -e '/^ *assert_decompile_roundtrip/d' \
 164       -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
 165   '';
 166
 167   preCheck = ''
 168     HOME="$TMPDIR"
 169     export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
 170
 171     # copy the test data to a directory
 172     ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
 173   '';
 174
 175   postCheck = ''
 176     rm -r "$IBIS_TEST_DATA_DIRECTORY"
 177   '';
 178
 179   pythonImportsCheck = [ "ibis" ] ++ map (backend: "ibis.backends.${backend}") testBackends;
 180
 181   optional-dependencies = {
 182     bigquery = [
 183       db-dtypes
 184       google-cloud-bigquery
 185       google-cloud-bigquery-storage
 186       pyarrow
 187       pyarrow-hotfix
 188       pydata-google-auth
 189       numpy
 190       pandas
 191       rich
 192     ];
 193     clickhouse = [
 194       clickhouse-connect
 195       pyarrow
 196       pyarrow-hotfix
 197       numpy
 198       pandas
 199       rich
 200     ];
 201     datafusion = [
 202       datafusion
 203       pyarrow
 204       pyarrow-hotfix
 205       numpy
 206       pandas
 207       rich
 208     ];
 209     druid = [
 210       pydruid
 211       pyarrow
 212       pyarrow-hotfix
 213       numpy
 214       pandas
 215       rich
 216     ];
 217     duckdb = [
 218       duckdb
 219       pyarrow
 220       pyarrow-hotfix
 221       numpy
 222       packaging
 223       pandas
 224       rich
 225     ];
 226     flink = [
 227       pyarrow
 228       pyarrow-hotfix
 229       numpy
 230       pandas
 231       rich
 232     ];
 233     geospatial = [
 234       geopandas
 235       shapely
 236     ];
 237     mssql = [
 238       pyodbc
 239       pyarrow
 240       pyarrow-hotfix
 241       numpy
 242       pandas
 243       rich
 244     ];
 245     mysql = [
 246       pymysql
 247       pyarrow
 248       pyarrow-hotfix
 249       numpy
 250       pandas
 251       rich
 252     ];
 253     oracle = [
 254       oracledb
 255       packaging
 256       pyarrow
 257       pyarrow-hotfix
 258       numpy
 259       pandas
 260       rich
 261     ];
 262     polars = [
 263       polars
 264       packaging
 265       pyarrow
 266       pyarrow-hotfix
 267       numpy
 268       pandas
 269       rich
 270     ];
 271     postgres = [
 272       psycopg2
 273       pyarrow
 274       pyarrow-hotfix
 275       numpy
 276       pandas
 277       rich
 278     ];
 279     pyspark = [
 280       pyspark
 281       packaging
 282       pyarrow
 283       pyarrow-hotfix
 284       numpy
 285       pandas
 286       rich
 287     ];
 288     snowflake = [
 289       snowflake-connector-python
 290       pyarrow
 291       pyarrow-hotfix
 292       numpy
 293       pandas
 294       rich
 295     ];
 296     sqlite = [
 297       regex
 298       pyarrow
 299       pyarrow-hotfix
 300       numpy
 301       pandas
 302       rich
 303     ];
 304     trino = [
 305       trino-python-client
 306       pyarrow
 307       pyarrow-hotfix
 308       numpy
 309       pandas
 310       rich
 311     ];
 312     visualization = [ graphviz ];
 313     decompiler = [ black ];
 314     examples = [ pins ] ++ pins.optional-dependencies.gcs;
 315   };
 316
 317   meta = with lib; {
 318     description = "Productivity-centric Python Big Data Framework";
 319     homepage = "https://github.com/ibis-project/ibis";
 320     changelog = "https://github.com/ibis-project/ibis/blob/${version}/docs/release_notes.md";
 321     license = licenses.asl20;
 322     maintainers = with maintainers; [ cpcloud ];
 323   };
 324 }