pkgs/development/python-modules/ibis-framework/default.nix

   1 { lib
   2 , buildPythonPackage
   3 , fetchFromGitHub
   4 , pythonOlder
   5 , pytestCheckHook
   6 , atpublic
   7 , bidict
   8 , black
   9 , clickhouse-connect
  10 , dask
  11 , datafusion
  12 , db-dtypes
  13 , duckdb
  14 , duckdb-engine
  15 , filelock
  16 , geoalchemy2
  17 , geopandas
  18 , google-cloud-bigquery
  19 , google-cloud-bigquery-storage
  20 , graphviz
  21 , hypothesis
  22 , multipledispatch
  23 , numpy
  24 , oracledb
  25 , packaging
  26 , pandas
  27 , parsy
  28 , pins
  29 , poetry-core
  30 , poetry-dynamic-versioning
  31 , polars
  32 , psycopg2
  33 , pyarrow
  34 , pyarrow-hotfix
  35 , pydata-google-auth
  36 , pydruid
  37 , pymysql
  38 , pyodbc
  39 , pyspark
  40 , pytest-benchmark
  41 , pytest-httpserver
  42 , pytest-mock
  43 , pytest-randomly
  44 , pytest-snapshot
  45 , pytest-xdist
  46 , python-dateutil
  47 , pytz
  48 , regex
  49 , rich
  50 , shapely
  51 , snowflake-connector-python
  52 , snowflake-sqlalchemy
  53 , sqlalchemy
  54 , sqlalchemy-views
  55 , sqlglot
  56 , sqlite
  57 , toolz
  58 , trino-python-client
  59 , typing-extensions
  60 }:
  61 let
  62   testBackends = [ "datafusion" "duckdb" "pandas" "sqlite" ];
  63
  64   ibisTestingData = fetchFromGitHub {
  65     name = "ibis-testing-data";
  66     owner = "ibis-project";
  67     repo = "testing-data";
  68     # https://github.com/ibis-project/ibis/blob/8.0.0/nix/overlay.nix#L20-L26
  69     rev = "2c6a4bb5d5d525058d8d5b2312a9fee5dafc5476";
  70     hash = "sha256-Lq503bqh9ESZJSk6yVq/uZwkAubzmSmoTBZSsqMm0DY=";
  71   };
  72 in
  73
  74 buildPythonPackage rec {
  75   pname = "ibis-framework";
  76   version = "8.0.0";
  77   pyproject = true;
  78
  79   disabled = pythonOlder "3.9";
  80
  81   src = fetchFromGitHub {
  82     name = "ibis-source";
  83     repo = "ibis";
  84     owner = "ibis-project";
  85     rev = "refs/tags/${version}";
  86     hash = "sha256-KcNZslqmSbu8uPYKpkyvd7d8Fsf0nQt80y0auXsI8fs=";
  87   };
  88
  89   nativeBuildInputs = [
  90     poetry-core
  91     poetry-dynamic-versioning
  92   ];
  93
  94   POETRY_DYNAMIC_VERSIONING_BYPASS = version;
  95
  96   propagatedBuildInputs = [
  97     atpublic
  98     bidict
  99     multipledispatch
 100     numpy
 101     pandas
 102     parsy
 103     pyarrow
 104     pyarrow-hotfix
 105     python-dateutil
 106     pytz
 107     rich
 108     sqlglot
 109     toolz
 110     typing-extensions
 111   ];
 112
 113   nativeCheckInputs = [
 114     pytestCheckHook
 115     black
 116     filelock
 117     hypothesis
 118     pytest-benchmark
 119     pytest-httpserver
 120     pytest-mock
 121     pytest-randomly
 122     pytest-snapshot
 123     pytest-xdist
 124   ] ++ lib.concatMap (name: passthru.optional-dependencies.${name}) testBackends;
 125
 126   pytestFlagsArray = [
 127     "--dist=loadgroup"
 128     "-m"
 129     "'${lib.concatStringsSep " or " testBackends} or core'"
 130   ];
 131
 132   disabledTests = [
 133     # breakage from sqlalchemy2 truediv changes
 134     "test_tpc_h17"
 135     # tries to download duckdb extensions
 136     "test_register_sqlite"
 137     "test_read_sqlite"
 138     # duckdb does not respect sample_size=2 (reads 3 lines of csv).
 139     "test_csv_reregister_schema"
 140     # duckdb fails with:
 141     # "This function can not be called with an active transaction!, commit or abort the existing one first"
 142     "test_vectorized_udf"
 143     "test_s3_403_fallback"
 144     "test_map_merge_udf"
 145     "test_udf"
 146     "test_map_udf"
 147     # DataFusion error
 148     "datafusion"
 149     # pluggy.PluggyTeardownRaisedWarning
 150     "test_repr_png_is_not_none_in_not_interactive"
 151     "test_interval_arithmetic"
 152   ];
 153
 154   # patch out tests that check formatting with black
 155   postPatch = ''
 156     find ibis/tests -type f -name '*.py' -exec sed -i \
 157       -e '/^ *assert_decompile_roundtrip/d' \
 158       -e 's/^\( *\)code = ibis.decompile(expr, format=True)/\1code = ibis.decompile(expr)/g' {} +
 159   '';
 160
 161   preCheck = ''
 162     HOME="$TMPDIR"
 163     export IBIS_TEST_DATA_DIRECTORY="ci/ibis-testing-data"
 164
 165     # copy the test data to a directory
 166     ln -s "${ibisTestingData}" "$IBIS_TEST_DATA_DIRECTORY"
 167   '';
 168
 169   postCheck = ''
 170     rm -r "$IBIS_TEST_DATA_DIRECTORY"
 171   '';
 172
 173   pythonImportsCheck = [
 174     "ibis"
 175   ] ++ map (backend: "ibis.backends.${backend}") testBackends;
 176
 177   passthru = {
 178     optional-dependencies = {
 179       bigquery = [ db-dtypes google-cloud-bigquery google-cloud-bigquery-storage pydata-google-auth ];
 180       clickhouse = [ clickhouse-connect sqlalchemy ];
 181       dask = [ dask regex ];
 182       datafusion = [ datafusion ];
 183       druid = [ pydruid sqlalchemy ];
 184       duckdb = [ duckdb duckdb-engine sqlalchemy sqlalchemy-views ];
 185       flink = [ ];
 186       geospatial = [ geoalchemy2 geopandas shapely ];
 187       mssql = [ sqlalchemy pyodbc sqlalchemy-views ];
 188       mysql = [ sqlalchemy pymysql sqlalchemy-views ];
 189       oracle = [ sqlalchemy oracledb packaging sqlalchemy-views ];
 190       pandas = [ regex ];
 191       polars = [ polars packaging ];
 192       postgres = [ psycopg2 sqlalchemy sqlalchemy-views ];
 193       pyspark = [ pyspark sqlalchemy packaging ];
 194       snowflake = [ snowflake-connector-python snowflake-sqlalchemy sqlalchemy-views packaging ];
 195       sqlite = [ regex sqlalchemy sqlalchemy-views ];
 196       trino = [ trino-python-client sqlalchemy sqlalchemy-views ];
 197       visualization = [ graphviz ];
 198       decompiler = [ black ];
 199       examples = [ pins ] ++ pins.optional-dependencies.gcs;
 200     };
 201   };
 202
 203   meta = with lib; {
 204     description = "Productivity-centric Python Big Data Framework";
 205     homepage = "https://github.com/ibis-project/ibis";
 206     changelog = "https://github.com/ibis-project/ibis/blob/${version}/docs/release_notes.md";
 207     license = licenses.asl20;
 208     maintainers = with maintainers; [ cpcloud ];
 209   };
 210 }