diff --git a/hstrat/_auxiliary_lib/_render_polars_snapshot.py b/hstrat/_auxiliary_lib/_render_polars_snapshot.py index f62374342..03b94317d 100644 --- a/hstrat/_auxiliary_lib/_render_polars_snapshot.py +++ b/hstrat/_auxiliary_lib/_render_polars_snapshot.py @@ -26,7 +26,7 @@ def render_polars_snapshot( The rendered DataFrame, as a string, if `display` is not used. """ with pl.Config() as cfg: - cfg.set_tbl_cols(df.lazy().collect_schema().len()) + cfg.set_tbl_cols(-1) head = repr(df.lazy().head().collect()) message = " ".join( [ diff --git a/hstrat/dataframe/_surface_unpack_reconstruct.py b/hstrat/dataframe/_surface_unpack_reconstruct.py index d9c523216..615b2843b 100644 --- a/hstrat/dataframe/_surface_unpack_reconstruct.py +++ b/hstrat/dataframe/_surface_unpack_reconstruct.py @@ -40,7 +40,14 @@ def _build_records_chunked( ): long_df = dstream_dataframe.explode_lookup_unpacked( df_slice, value_type="uint64" - ).select( + ) + + with log_context_duration( + '.sort_by("dstream_Tbar").over(partition_by="dstream_data_id") ' + f"({i + 1}/{num_slices})", + logging.info, + ): + long_df = long_df.select( pl.col( "dstream_data_id", "dstream_T", @@ -264,7 +271,7 @@ def surface_unpack_reconstruct( res["dstream_S"] = pd.Series(dtype=int) return pl.from_pandas(res) - logging.info("extracting metadata..") + logging.info("extracting metadata...") dstream_storage_bitwidth = get_sole_scalar_value_polars( df, "dstream_storage_bitwidth" ) diff --git a/pyproject.toml b/pyproject.toml index fb0fc361b..b064ebfca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ dependencies = [ "bitstring>=3.1.9", "dendropy>=4.5.2", "Deprecated>=1.2.13", - "downstream>=1.4.0", + "downstream>=1.5.1", "iterpop>=0.3.4", "interval_search>=0.3.1", "joinem>=0.9.1", diff --git a/requirements-dev/py310/requirements-all.txt b/requirements-dev/py310/requirements-all.txt index 42895084a..46cce7a10 100644 --- a/requirements-dev/py310/requirements-all.txt +++ b/requirements-dev/py310/requirements-all.txt @@ -79,7 +79,7 @@ docutils==0.21.2 # readme-renderer # sphinx # sphinx-rtd-theme -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) entrypoints==0.3 # via flake8 @@ -304,6 +304,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) prompt-toolkit==3.0.48 diff --git a/requirements-dev/py310/requirements-docs.txt b/requirements-dev/py310/requirements-docs.txt index 86de36926..202aeb498 100644 --- a/requirements-dev/py310/requirements-docs.txt +++ b/requirements-dev/py310/requirements-docs.txt @@ -67,7 +67,7 @@ docutils==0.21.2 # nbsphinx # sphinx # sphinx-rtd-theme -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -239,6 +239,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) prompt-toolkit==3.0.48 diff --git a/requirements-dev/py310/requirements-jit.txt b/requirements-dev/py310/requirements-jit.txt index 91641e220..55ffd8dd3 100644 --- a/requirements-dev/py310/requirements-jit.txt +++ b/requirements-dev/py310/requirements-jit.txt @@ -36,7 +36,7 @@ deprecated==1.2.15 # via # hstrat (../../pyproject.toml) # alifedata-phyloinformatics-convert -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -145,6 +145,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py310/requirements-minimal.txt b/requirements-dev/py310/requirements-minimal.txt index 043ac66e9..3f15e0d67 100644 --- a/requirements-dev/py310/requirements-minimal.txt +++ b/requirements-dev/py310/requirements-minimal.txt @@ -34,7 +34,7 @@ deprecated==1.2.15 # via # hstrat (../../pyproject.toml) # alifedata-phyloinformatics-convert -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -132,6 +132,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py310/requirements-release.txt b/requirements-dev/py310/requirements-release.txt index 139bee5ea..14cc613d2 100644 --- a/requirements-dev/py310/requirements-release.txt +++ b/requirements-dev/py310/requirements-release.txt @@ -45,7 +45,7 @@ deprecated==1.2.15 # alifedata-phyloinformatics-convert docutils==0.21.2 # via readme-renderer -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -154,6 +154,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py310/requirements-testing.txt b/requirements-dev/py310/requirements-testing.txt index 9daa24b86..a522e317f 100644 --- a/requirements-dev/py310/requirements-testing.txt +++ b/requirements-dev/py310/requirements-testing.txt @@ -40,7 +40,7 @@ deprecated==1.2.15 # alifedata-phyloinformatics-convert distlib==0.3.9 # via virtualenv -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) entrypoints==0.3 # via flake8 @@ -183,6 +183,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py311/requirements-all.txt b/requirements-dev/py311/requirements-all.txt index 756e8b17b..3097e0eb1 100644 --- a/requirements-dev/py311/requirements-all.txt +++ b/requirements-dev/py311/requirements-all.txt @@ -79,7 +79,7 @@ docutils==0.21.2 # readme-renderer # sphinx # sphinx-rtd-theme -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) entrypoints==0.3 # via flake8 @@ -302,6 +302,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) prompt-toolkit==3.0.48 diff --git a/requirements-dev/py311/requirements-docs.txt b/requirements-dev/py311/requirements-docs.txt index 92b52cd7f..ce902807f 100644 --- a/requirements-dev/py311/requirements-docs.txt +++ b/requirements-dev/py311/requirements-docs.txt @@ -67,7 +67,7 @@ docutils==0.21.2 # nbsphinx # sphinx # sphinx-rtd-theme -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -239,6 +239,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) prompt-toolkit==3.0.48 diff --git a/requirements-dev/py311/requirements-jit.txt b/requirements-dev/py311/requirements-jit.txt index 929db4d55..f8488bbe0 100644 --- a/requirements-dev/py311/requirements-jit.txt +++ b/requirements-dev/py311/requirements-jit.txt @@ -36,7 +36,7 @@ deprecated==1.2.15 # via # hstrat (../../pyproject.toml) # alifedata-phyloinformatics-convert -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -145,6 +145,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py311/requirements-minimal.txt b/requirements-dev/py311/requirements-minimal.txt index e1a678439..d883f78ca 100644 --- a/requirements-dev/py311/requirements-minimal.txt +++ b/requirements-dev/py311/requirements-minimal.txt @@ -34,7 +34,7 @@ deprecated==1.2.15 # via # hstrat (../../pyproject.toml) # alifedata-phyloinformatics-convert -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -132,6 +132,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py311/requirements-release.txt b/requirements-dev/py311/requirements-release.txt index b96a9bba4..8aa8636e6 100644 --- a/requirements-dev/py311/requirements-release.txt +++ b/requirements-dev/py311/requirements-release.txt @@ -45,7 +45,7 @@ deprecated==1.2.15 # alifedata-phyloinformatics-convert docutils==0.21.2 # via readme-renderer -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -154,6 +154,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py311/requirements-testing.txt b/requirements-dev/py311/requirements-testing.txt index 4ad5ed04e..45b27978a 100644 --- a/requirements-dev/py311/requirements-testing.txt +++ b/requirements-dev/py311/requirements-testing.txt @@ -40,7 +40,7 @@ deprecated==1.2.15 # alifedata-phyloinformatics-convert distlib==0.3.9 # via virtualenv -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) entrypoints==0.3 # via flake8 @@ -181,6 +181,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py312/requirements-all.txt b/requirements-dev/py312/requirements-all.txt index 756e8b17b..3097e0eb1 100644 --- a/requirements-dev/py312/requirements-all.txt +++ b/requirements-dev/py312/requirements-all.txt @@ -79,7 +79,7 @@ docutils==0.21.2 # readme-renderer # sphinx # sphinx-rtd-theme -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) entrypoints==0.3 # via flake8 @@ -302,6 +302,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) prompt-toolkit==3.0.48 diff --git a/requirements-dev/py312/requirements-docs.txt b/requirements-dev/py312/requirements-docs.txt index 92b52cd7f..ce902807f 100644 --- a/requirements-dev/py312/requirements-docs.txt +++ b/requirements-dev/py312/requirements-docs.txt @@ -67,7 +67,7 @@ docutils==0.21.2 # nbsphinx # sphinx # sphinx-rtd-theme -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -239,6 +239,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) prompt-toolkit==3.0.48 diff --git a/requirements-dev/py312/requirements-jit.txt b/requirements-dev/py312/requirements-jit.txt index 929db4d55..2ca2e7e5f 100644 --- a/requirements-dev/py312/requirements-jit.txt +++ b/requirements-dev/py312/requirements-jit.txt @@ -1,5 +1,5 @@ # This file was autogenerated by uv via the following command: -# uv pip compile --python-version=3.11 ../../pyproject.toml --extra jit -o requirements-jit.txt +# uv pip compile --python-version=3.12 ../../pyproject.toml --extra jit -o requirements-jit.txt alifedata-phyloinformatics-convert==0.17.0 # via hstrat (../../pyproject.toml) annotated-types==0.7.0 @@ -36,7 +36,7 @@ deprecated==1.2.15 # via # hstrat (../../pyproject.toml) # alifedata-phyloinformatics-convert -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -145,6 +145,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py312/requirements-minimal.txt b/requirements-dev/py312/requirements-minimal.txt index e1a678439..d883f78ca 100644 --- a/requirements-dev/py312/requirements-minimal.txt +++ b/requirements-dev/py312/requirements-minimal.txt @@ -34,7 +34,7 @@ deprecated==1.2.15 # via # hstrat (../../pyproject.toml) # alifedata-phyloinformatics-convert -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -132,6 +132,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py312/requirements-release.txt b/requirements-dev/py312/requirements-release.txt index b96a9bba4..8aa8636e6 100644 --- a/requirements-dev/py312/requirements-release.txt +++ b/requirements-dev/py312/requirements-release.txt @@ -45,7 +45,7 @@ deprecated==1.2.15 # alifedata-phyloinformatics-convert docutils==0.21.2 # via readme-renderer -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) ete3==3.1.3 # via alifedata-phyloinformatics-convert @@ -154,6 +154,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/requirements-dev/py312/requirements-testing.txt b/requirements-dev/py312/requirements-testing.txt index 4ad5ed04e..45b27978a 100644 --- a/requirements-dev/py312/requirements-testing.txt +++ b/requirements-dev/py312/requirements-testing.txt @@ -40,7 +40,7 @@ deprecated==1.2.15 # alifedata-phyloinformatics-convert distlib==0.3.9 # via virtualenv -downstream==1.4.0 +downstream==1.5.1 # via hstrat (../../pyproject.toml) entrypoints==0.3 # via flake8 @@ -181,6 +181,8 @@ polars==1.16.0 # hstrat (../../pyproject.toml) # downstream # joinem +polars-u64-idx==1.18.0 + # via hstrat (../../pyproject.toml) prettytable==3.12.0 # via hstrat (../../pyproject.toml) propcache==0.2.0 diff --git a/tests/test_hstrat/test_dataframe/assets/packed.csv b/tests/test_hstrat/test_dataframe/assets/packed.csv index f24713039..54abd2edb 100644 --- a/tests/test_hstrat/test_dataframe/assets/packed.csv +++ b/tests/test_hstrat/test_dataframe/assets/packed.csv @@ -1,3 +1,3 @@ awoo,dstream_algo,downstream_version,data_hex,dstream_storage_bitoffset,dstream_storage_bitwidth,dstream_T_bitoffset,dstream_T_bitwidth,dstream_S -bar,dstream.steady_algo,1.0.1,080001030702050406,8,96,0,8,8 -baz,dstream.steady_algo,1.0.1,0b0001030702050906,8,96,0,8,8 +bar,dstream.steady_algo,1.0.1,080001030702050406,8,64,0,8,8 +baz,dstream.steady_algo,1.0.1,0b0001030702050906,8,64,0,8,8