add text/llm+plain media type (#281)

* add repr_llm dependency

* add text/llm+plain with summary

* remove 3.8

* don't add text/llm+plain if we hit a summarize error

.github/workflows/ci.yaml

@@ -13,7 +13,7 @@ jobs:
  runs-on: ubuntu-20.04
  strategy:
  matrix:
- python_version: ["3.8", "3.9", "3.10"]
+ python_version: ["3.9", "3.10"]
  steps:
  - name: Checkout code
  uses: actions/checkout@v3
@@ -22,7 +22,7 @@ jobs:
  - name: Setup poetry
  uses: Gr1N/setup-poetry@v8
  with:
- poetry-version: "1.4.1"
+ poetry-version: "1.4.2"
  - name: Install dependencies
  run: |
  poetry export -f requirements.txt --extras "docs" --without-hashes --with dev --output requirements.txt

pyproject.toml

@@ -2,26 +2,30 @@
 name = "dx"
 version = "1.3.0"
 description = "Python wrapper for Data Explorer"
-authors = ["Dave Shoup <[email protected]>", "Kyle Kelley <[email protected]>"]
+authors = [
+ "Dave Shoup <[email protected]>",
+ "Kyle Kelley <[email protected]>",
+]
 readme = "README.md"
 license = "MIT"
 homepage = "https://app.noteable.io/"
 repository = "https://github.com/noteable-io/dx"
 keywords = ["data", "exploration", "visualization"]
 
 [tool.poetry.dependencies]
-python = "^3.8"
+python = "^3.9"
 pandas = "^1.3.5"
 ipython = ">=7.31.1"
 pydantic = "^1.9"
 structlog = "^22.1.0"
-mkdocs = {version = "^1.3.1", optional = true}
-mkdocs-material = {version = "^8.3.9", optional = true}
-mkdocs-jupyter = {version = ">=0.21,<0.23", optional = true}
-mkdocstrings = {version = ">=0.19,<0.22", optional = true}
-mkdocstrings-python = {version = ">=0.7.1,<0.10.0", optional = true}
+mkdocs = { version = "^1.3.1", optional = true }
+mkdocs-material = { version = "^8.3.9", optional = true }
+mkdocs-jupyter = { version = ">=0.21,<0.23", optional = true }
+mkdocstrings = { version = ">=0.19,<0.22", optional = true }
+mkdocstrings-python = { version = ">=0.7.1,<0.10.0", optional = true }
 duckdb-engine = ">=0.6.4,<0.8.0"
 exceptiongroup = "^1.0.4"
+repr-llm = "^0.2.1"
 
 [tool.poetry.group.dev.dependencies]
 black = ">=22.12,<24.0"
@@ -38,12 +42,19 @@ pytest-mock = "^3.10.0"
 Faker = ">=15.3.4,<19.0.0"
 geopandas = "^0.12.2"
 polars = ">=0.16.12,<0.18.0"
-modin = {extras = ["all"], version = "^0.19.0"}
+modin = { extras = ["all"], version = "^0.19.0" }
 dask = "^2023.3.1"
 vaex = "^4.16.0"
 
 [tool.poetry.extras]
-docs = ["mkdocs", "mkdocs-material", "mkdocs-jupyter", "mkdocstrings", "mkdocstrings-python", "mkdocs-glightbox"]
+docs = [
+ "mkdocs",
+ "mkdocs-material",
+ "mkdocs-jupyter",
+ "mkdocstrings",
+ "mkdocstrings-python",
+ "mkdocs-glightbox",
+]
 
 [build-system]
 requires = ["poetry_core>=1.0.0"]
@@ -61,10 +72,12 @@ branch = false
 omit = ["*/tests/*"]
 
 [tool.coverage.report]
-exclude_lines = ["if self.debug:",
- "pragma: no cover",
- "raise AssertionError",
- "raise NotImplementedError",
- "if __name__ == '__main__':"]
+exclude_lines = [
+ "if self.debug:",
+ "pragma: no cover",
+ "raise AssertionError",
+ "raise NotImplementedError",
+ "if __name__ == '__main__':",
+]
 ignore_errors = true
 omit = []

src/dx/formatters/main.py

@@ -9,6 +9,7 @@
 from IPython.core.interactiveshell import InteractiveShell
 from IPython.display import display as ipydisplay
 from pandas.io.json import build_table_schema
+from repr_llm.pandas import summarize_dataframe
 
 from dx.sampling import get_column_string_lengths, get_df_dimensions, sample_if_too_big
 from dx.settings import get_settings
@@ -212,6 +213,13 @@ def format_output(
  )
 
  payload = {settings.MEDIA_TYPE: payload}
+ # add additional payload for LLM consumption; if any parsing/summarizing errors occur, we
+ # shouldn't block displaying the bundle
+ try:
+ payload["text/llm+plain"] = summarize_dataframe(df)
+ except Exception as e:
+ logger.debug(f"Error in summarize_dataframe: {e}")
+
  metadata = {settings.MEDIA_TYPE: metadata}
 
  # this needs to happen so we can update by display_id as needed