pkgs/development/python-modules/ibis-framework/default.nix

   1 {
   2   lib,
   3   buildPythonPackage,
   4   fetchFromGitHub,
   5   pythonOlder,
   6   pytestCheckHook,
   7   atpublic,
   8   black,
   9   clickhouse-connect,
  10   datafusion,
  11   db-dtypes,
  12   duckdb,
  13   fetchpatch,
  14   filelock,
  15   geopandas,
  16   google-cloud-bigquery,
  17   google-cloud-bigquery-storage,
  18   graphviz,
  19   hypothesis,
  20   numpy,
  21   oracledb,
  22   packaging,
  23   pandas,
  24   parsy,
  25   pins,
  26   poetry-core,
  27   poetry-dynamic-versioning,
  28   polars,
  29   psycopg2,
  30   pyarrow,
  31   pyarrow-hotfix,
  32   pydata-google-auth,
  33   pydruid,
  34   pymysql,
  35   pyodbc,
  36   pyspark,
  37   pytest-benchmark,
  38   pytest-httpserver,
  39   pytest-mock,
  40   pytest-randomly,
  41   pytest-snapshot,
  42   pytest-timeout,
  43   pytest-xdist,
  44   python-dateutil,
  45   pytz,
  46   regex,
  47   rich,
  48   shapely,
  49   snowflake-connector-python,
  50   sqlglot,
  51   sqlite,
  52   toolz,
  53   trino-python-client,
  54   typing-extensions,
  55 }:
  56 let
  57   testBackends = [
  58     "duckdb"
  59     "sqlite"
  60   ];
  61
  62   ibisTestingData = fetchFromGitHub {
  63     name = "ibis-testing-data";
  64     owner = "ibis-project";
  65     repo = "testing-data";
  66     # https://github.com/ibis-project/ibis/blob/9.5.0/nix/overlay.nix#L20-L26
  67     rev = "b26bd40cf29004372319df620c4bbe41420bb6f8";
  68     hash = "sha256-1fenQNQB+Q0pbb0cbK2S/UIwZDE4PXXG15MH3aVbyLU=";
  69   };
  70 in
  71
  72 buildPythonPackage rec {
  73   pname = "ibis-framework";
  74   version = "9.5.0";
  75   pyproject = true;
  76
  77   disabled = pythonOlder "3.10";
  78
  79   src = fetchFromGitHub {
  80     name = "ibis-source";
  81     repo = "ibis";
  82     owner = "ibis-project";
  83     rev = "refs/tags/${version}";
  84     hash = "sha256-6ebw/E3jZFMHKqC5ZY//2Ke0NrklyoGp5JGKBfDxy40=";
  85   };
  86
  87   patches = [
  88     # remove after the 10.0 release
  89     (fetchpatch {
  90       name = "ibis-framework-duckdb-1.1.1.patch";
  91       url = "https://github.com/ibis-project/ibis/commit/a54eceabac1d6592e9f6ab0ca7749e37a748c2ad.patch";
  92       hash = "sha256-j5BPYVqnEF9GQV5N3/VhFUCdsEwAIOQC0KfZ5LNBSRg=";
  93     })
  94
  95     # remove after the 10.0 release
  96     (fetchpatch {
  97       name = "ibis-framework-arrow-18.patch";
  98       url = "https://github.com/ibis-project/ibis/commit/5dc549b22c2eca29a11a31fb29deef7c1466a204.patch";
  99       hash = "sha256-4i/g2uixdlkbE6x659wzZJ91FZpzwOVkF6ZeXkiCP3I=";
 100       excludes = [
 101         "poetry.lock"
 102         "requirements-dev.txt"
 103       ];
 104     })
 105   ];
 106
 107   build-system = [
 108     poetry-core
 109     poetry-dynamic-versioning
 110   ];
 111
 112   dontBypassPoetryDynamicVersioning = true;
 113   env.POETRY_DYNAMIC_VERSIONING_BYPASS = lib.head (lib.strings.splitString "-" version);
 114
 115   dependencies = [
 116     atpublic
 117     parsy
 118     python-dateutil
 119     pytz
 120     sqlglot
 121     toolz
 122     typing-extensions
 123   ];
 124
 125   nativeCheckInputs = [
 126     pytestCheckHook
 127     black
 128     filelock
 129     hypothesis
 130     pytest-benchmark
 131     pytest-httpserver
 132     pytest-mock
 133     pytest-randomly
 134     pytest-snapshot
 135     pytest-timeout
 136     # this dependency is still needed due to use of strict markers and
 137     # `pytest.mark.xdist_group` in the ibis codebase
 138     pytest-xdist
 139   ] ++ lib.concatMap (name: optional-dependencies.${name}) testBackends;
 140
 141   pytestFlagsArray = [
 142     "-m"
 143     "'${lib.concatStringsSep " or " testBackends} or core'"
 144   ];
 145
 146   disabledTests = [
 147     # tries to download duckdb extensions
 148     "test_attach_sqlite"
 149     "test_connect_extensions"
 150     "test_load_extension"
 151     "test_read_sqlite"
 152     "test_register_sqlite"
 153     # requires network connection
 154     "test_s3_403_fallback"
 155     "test_hugging_face"
 156     # requires pytest 8.2+
 157     "test_roundtrip_delta"
 158   ];
 159
 160   # patch out tests that check formatting with black
 161   postPatch = ''
 162     find ibis/tests -type f -name '*.py' -exec sed -i \
 163       -e '/^ *assert_decompile_roundtrip/d' \
 164       -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
 165   '';
 166
 167   preCheck = ''
 168     HOME="$TMPDIR"
 169     export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
 170
 171     # copy the test data to a directory
 172     ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
 173   '';
 174
 175   postCheck = ''
 176     rm -r "$IBIS_TEST_DATA_DIRECTORY"
 177   '';
 178
 179   pythonImportsCheck = [ "ibis" ] ++ map (backend: "ibis.backends.${backend}") testBackends;
 180
 181   optional-dependencies = {
 182     bigquery = [
 183       db-dtypes
 184       google-cloud-bigquery
 185       google-cloud-bigquery-storage
 186       pyarrow
 187       pyarrow-hotfix
 188       pydata-google-auth
 189       numpy
 190       pandas
 191       rich
 192     ];
 193     clickhouse = [
 194       clickhouse-connect
 195       pyarrow
 196       pyarrow-hotfix
 197       numpy
 198       pandas
 199       rich
 200     ];
 201     datafusion = [
 202       datafusion
 203       pyarrow
 204       pyarrow-hotfix
 205       numpy
 206       pandas
 207       rich
 208     ];
 209     druid = [
 210       pydruid
 211       pyarrow
 212       pyarrow-hotfix
 213       numpy
 214       pandas
 215       rich
 216     ];
 217     duckdb = [
 218       duckdb
 219       pyarrow
 220       pyarrow-hotfix
 221       numpy
 222       pandas
 223       rich
 224     ];
 225     flink = [
 226       pyarrow
 227       pyarrow-hotfix
 228       numpy
 229       pandas
 230       rich
 231     ];
 232     geospatial = [
 233       geopandas
 234       shapely
 235     ];
 236     mssql = [
 237       pyodbc
 238       pyarrow
 239       pyarrow-hotfix
 240       numpy
 241       pandas
 242       rich
 243     ];
 244     mysql = [
 245       pymysql
 246       pyarrow
 247       pyarrow-hotfix
 248       numpy
 249       pandas
 250       rich
 251     ];
 252     oracle = [
 253       oracledb
 254       packaging
 255       pyarrow
 256       pyarrow-hotfix
 257       numpy
 258       pandas
 259       rich
 260     ];
 261     polars = [
 262       polars
 263       packaging
 264       pyarrow
 265       pyarrow-hotfix
 266       numpy
 267       pandas
 268       rich
 269     ];
 270     postgres = [
 271       psycopg2
 272       pyarrow
 273       pyarrow-hotfix
 274       numpy
 275       pandas
 276       rich
 277     ];
 278     pyspark = [
 279       pyspark
 280       packaging
 281       pyarrow
 282       pyarrow-hotfix
 283       numpy
 284       pandas
 285       rich
 286     ];
 287     snowflake = [
 288       snowflake-connector-python
 289       pyarrow
 290       pyarrow-hotfix
 291       numpy
 292       pandas
 293       rich
 294     ];
 295     sqlite = [
 296       regex
 297       pyarrow
 298       pyarrow-hotfix
 299       numpy
 300       pandas
 301       rich
 302     ];
 303     trino = [
 304       trino-python-client
 305       pyarrow
 306       pyarrow-hotfix
 307       numpy
 308       pandas
 309       rich
 310     ];
 311     visualization = [ graphviz ];
 312     decompiler = [ black ];
 313     examples = [ pins ] ++ pins.optional-dependencies.gcs;
 314   };
 315
 316   meta = with lib; {
 317     description = "Productivity-centric Python Big Data Framework";
 318     homepage = "https://github.com/ibis-project/ibis";
 319     changelog = "https://github.com/ibis-project/ibis/blob/${version}/docs/release_notes.md";
 320     license = licenses.asl20;
 321     maintainers = with maintainers; [ cpcloud ];
 322   };
 323 }