Fix: Resolve Python 3.9 compatibility issues, expand CI test matrix (#98

)

.github/workflows/python_pytest.yml

@@ -48,10 +48,9 @@ jobs:
     strategy:
       matrix:
         python-version: [
-          # TODO: Re-enable 3.9 and 3.11 once we have stable tests across all versions.
-          # '3.9',
+          '3.9',
           '3.10',
-          # '3.11',
+          '3.11',
         ]
       fail-fast: false
 

airbyte/_util/document_rendering.py

@@ -2,7 +2,7 @@
 """Methods for converting Airbyte records into documents."""
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional
 
 import yaml
 from pydantic import BaseModel
@@ -27,7 +27,7 @@ def _to_title_case(name: str, /) -> str:
 class CustomRenderingInstructions(BaseModel):
     """Instructions for rendering a stream's records as documents."""
 
-    title_property: str | None
+    title_property: Optional[str]
     content_properties: list[str]
     frontmatter_properties: list[str]
     metadata_properties: list[str]
@@ -36,9 +36,9 @@ class CustomRenderingInstructions(BaseModel):
 class DocumentRenderer(BaseModel):
     """Instructions for rendering a stream's records as documents."""
 
-    title_property: str | None
-    content_properties: list[str] | None
-    metadata_properties: list[str] | None
+    title_property: Optional[str]
+    content_properties: Optional[list[str]]
+    metadata_properties: Optional[list[str]]
     render_metadata: bool = False
 
     # TODO: Add primary key and cursor key support:

airbyte/caches/base.py

@@ -5,7 +5,7 @@
 
 import abc
 from pathlib import Path
-from typing import TYPE_CHECKING, Any, final
+from typing import TYPE_CHECKING, Any, Optional, final
 
 from pydantic import BaseModel, PrivateAttr
 
@@ -34,7 +34,7 @@ class CacheBase(BaseModel):
     schema_name: str = "airbyte_raw"
     """The name of the schema to write to."""
 
-    table_prefix: str | None = None
+    table_prefix: Optional[str] = None
     """ A prefix to add to all table names.
     If 'None', a prefix will be created based on the source name.
     """
@@ -43,7 +43,7 @@ class CacheBase(BaseModel):
     """A suffix to add to all table names."""
 
     _sql_processor_class: type[SqlProcessorBase] = PrivateAttr()
-    _sql_processor: SqlProcessorBase | None = PrivateAttr(default=None)
+    _sql_processor: Optional[SqlProcessorBase] = PrivateAttr(default=None)
 
     @final
     @property

airbyte/caches/duckdb.py

@@ -5,6 +5,7 @@
 
 import warnings
 from pathlib import Path  # noqa: TCH003  # Used in Pydantic init
+from typing import Union
 
 from overrides import overrides
 
@@ -23,7 +24,7 @@
 class DuckDBCache(CacheBase):
     """A DuckDB cache."""
 
-    db_path: Path | str
+    db_path: Union[Path, str]
     """Normally db_path is a Path object.
 
     There are some cases, such as when connecting to MotherDuck, where it could be a string that

airbyte/documents.py

@@ -2,9 +2,9 @@
 """Methods for converting Airbyte records into documents."""
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, Optional
 
-from pydantic import BaseModel
+from pydantic import BaseModel, Field
 
 
 if TYPE_CHECKING:
@@ -30,10 +30,10 @@ class Document(BaseModel):
     This class is duck-typed to be compatible with LangChain project's `Document` class.
     """
 
-    id: str | None = None
+    id: Optional[str] = Field(default=None)
     content: str
     metadata: dict[str, Any]
-    last_modified: datetime.datetime | None = None
+    last_modified: Optional[datetime.datetime] = Field(default=None)
 
     def __str__(self) -> str:
         return self.content

examples/run_file_source.py

@@ -0,0 +1,23 @@
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+"""A simple test of PyAirbyte, using the File source connector.
+
+Usage (from PyAirbyte root directory):
+> poetry run python ./examples/run_file.py
+
+No setup is needed, but you may need to delete the .venv-source-file folder
+if your installation gets interrupted or corrupted.
+"""
+
+from __future__ import annotations
+
+import airbyte as ab
+
+
+source = ab.get_source(
+    "source-file",
+    install_if_missing=True,
+)
+source.check()
+
+# print(list(source.get_records("pokemon")))
+source.read(cache=ab.new_local_cache("poke"))

pyproject.toml

@@ -164,6 +164,8 @@ ignore = [
     "FIX002", # Allow "TODO:" until release (then switch to requiring links via TDO003)
     "PLW0603", # Using the global statement to update _cache is discouraged
     "TD003", # Require links for TODOs # TODO: Re-enable when we disable FIX002
+
+    "UP007", # Allow legacy `Union[a, b]` and `Optional[a]` for Pydantic, until we drop Python 3.9 (Pydantic doesn't like it)
 ]
 fixable = ["ALL"]
 unfixable = [

tests/conftest.py

@@ -1,6 +1,6 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
-
 """Global pytest fixtures."""
+from __future__ import annotations
 
 import json
 import logging

tests/integration_tests/fixtures/source-broken/setup.py

@@ -1,7 +1,7 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+from __future__ import annotations
 
 from setuptools import setup
 

tests/integration_tests/fixtures/source-broken/source_broken/run.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 def run():
-    raise Exception("Could not run")
+    raise Exception("Could not run")

tests/integration_tests/fixtures/source-test/setup.py

@@ -1,14 +1,14 @@
 #
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
 #
-
+from __future__ import annotations
 
 from setuptools import setup
 
 setup(
     name="airbyte-source-test",
     version="0.0.1",
-    description="Test Soutce",
+    description="Test Source",
     author="Airbyte",
     author_email="[email protected]",
     packages=["source_test"],

tests/integration_tests/fixtures/source-test/source_test/run.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import json
 import sys

tests/integration_tests/test_install.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 from gettext import install
 import pytest

tests/integration_tests/test_snowflake_cache.py

@@ -6,6 +6,7 @@
 and available on PATH for the poetry-managed venv.
 """
 from __future__ import annotations
+
 from collections.abc import Generator
 import os
 import sys

tests/integration_tests/test_source_test_fixture.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 from collections.abc import Mapping
 import os

tests/integration_tests/test_validation.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import os
 import shutil

tests/lint_tests/test_mypy.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import subprocess
 

tests/lint_tests/test_ruff.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import subprocess
 

tests/unit_tests/test_anonymous_usage_stats.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import itertools
 from contextlib import nullcontext as does_not_raise

tests/unit_tests/test_caches.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 from pathlib import Path
 

tests/unit_tests/test_exceptions.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import inspect
 import pytest

tests/unit_tests/test_pip_helpers.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import pytest
 from airbyte._util import github_pip_url, connector_pip_url

tests/unit_tests/test_progress.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import datetime
 from textwrap import dedent

tests/unit_tests/test_type_translation.py

@@ -1,4 +1,5 @@
 # Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+from __future__ import annotations
 
 import pytest
 from sqlalchemy import types