From c47a423594237a0e442760b6a295cb41962b86dc Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:08:16 -0800 Subject: [PATCH 01/28] add github ci workflows --- .github/workflows/python_autofix.yml | 40 ++++++++++++++++++++++ .github/workflows/python_format_checks.yml | 31 +++++++++++++++++ .github/workflows/python_lint.yml | 40 ++++++++++++++++++++++ 3 files changed, 111 insertions(+) create mode 100644 .github/workflows/python_autofix.yml create mode 100644 .github/workflows/python_format_checks.yml create mode 100644 .github/workflows/python_lint.yml diff --git a/.github/workflows/python_autofix.yml b/.github/workflows/python_autofix.yml new file mode 100644 index 00000000..cf1aa99c --- /dev/null +++ b/.github/workflows/python_autofix.yml @@ -0,0 +1,40 @@ +name: Auto-fix Python + +on: + workflow_dispatch: + +jobs: + format-fix-and-check: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + cache: 'poetry' + + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" + + - name: Install dependencies + run: poetry install + + - name: Format code + run: poetry run ruff format . + + - name: Commit changes + run: | + git config --global user.name "Airbyte Automation Bot" + git config --global user.email "no-reply@airbyte.io" + git add . + git diff-index --quiet HEAD || git commit -m "Format code with black" + + - name: Push changes + uses: ad-m/github-push-action@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/python_format_checks.yml b/.github/workflows/python_format_checks.yml new file mode 100644 index 00000000..3daf1f26 --- /dev/null +++ b/.github/workflows/python_format_checks.yml @@ -0,0 +1,31 @@ +name: Check Python Format + +on: + push: + branches: + - main + pull_request: + +jobs: + format-check: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + cache: 'poetry' + + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" + + - name: Install dependencies + run: poetry install + + - name: Check code format + run: poetry run ruff format --check . diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml new file mode 100644 index 00000000..f5a70610 --- /dev/null +++ b/.github/workflows/python_lint.yml @@ -0,0 +1,40 @@ +name: Lint Python + +on: + push: + branches: + - main + pull_request: + +jobs: + format-check: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + cache: 'poetry' + + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" + + - name: Install dependencies + run: poetry install + + - name: Format code + run: poetry run ruff format --check . + + - name: Raise error if formatting issues found + run: | + if [ $? -eq 0 ]; then + echo "No formatting issues found." + else + echo "Formatting issues found. Please run 'poetry run ruff format --check .' to fix them." + exit 1 + fi From f33393aa61356253cf6e9725dc04fb1b2628fbd0 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:10:53 -0800 Subject: [PATCH 02/28] fix lint --- .github/workflows/python_lint.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index f5a70610..12ab379e 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -12,17 +12,16 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" - name: Set up Python uses: actions/setup-python@v5 with: python-version: 3.9 cache: 'poetry' - - name: Set up Poetry - uses: Gr1N/setup-poetry@v8 - with: - poetry-version: "1.7.1" - name: Install dependencies run: poetry install From 86568887a73a4d266d51db2912e8d1524108091f Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:13:50 -0800 Subject: [PATCH 03/28] fix bugs --- .github/workflows/python_format_checks.yml | 10 ++++------ .github/workflows/python_lint.yml | 12 +----------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/.github/workflows/python_format_checks.yml b/.github/workflows/python_format_checks.yml index 3daf1f26..77b1279e 100644 --- a/.github/workflows/python_format_checks.yml +++ b/.github/workflows/python_format_checks.yml @@ -12,18 +12,16 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" - name: Set up Python uses: actions/setup-python@v5 with: python-version: 3.9 cache: 'poetry' - - name: Set up Poetry - uses: Gr1N/setup-poetry@v8 - with: - poetry-version: "1.7.1" - - name: Install dependencies run: poetry install diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index 12ab379e..e4c63a69 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -22,18 +22,8 @@ jobs: python-version: 3.9 cache: 'poetry' - - name: Install dependencies run: poetry install - name: Format code - run: poetry run ruff format --check . - - - name: Raise error if formatting issues found - run: | - if [ $? -eq 0 ]; then - echo "No formatting issues found." - else - echo "Formatting issues found. Please run 'poetry run ruff format --check .' to fix them." - exit 1 - fi + run: poetry run ruff check . From 672c2f472e1ace9bfd866077b584fd18bbd403ee Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:17:20 -0800 Subject: [PATCH 04/28] fix job names --- .github/workflows/python_autofix.yml | 2 +- .github/workflows/python_format_checks.yml | 2 +- .github/workflows/python_lint.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python_autofix.yml b/.github/workflows/python_autofix.yml index cf1aa99c..2d4a9e15 100644 --- a/.github/workflows/python_autofix.yml +++ b/.github/workflows/python_autofix.yml @@ -4,7 +4,7 @@ on: workflow_dispatch: jobs: - format-fix-and-check: + python-autofix: runs-on: ubuntu-latest steps: - name: Checkout code diff --git a/.github/workflows/python_format_checks.yml b/.github/workflows/python_format_checks.yml index 77b1279e..05331e05 100644 --- a/.github/workflows/python_format_checks.yml +++ b/.github/workflows/python_format_checks.yml @@ -7,7 +7,7 @@ on: pull_request: jobs: - format-check: + python-format-check: runs-on: ubuntu-latest steps: - name: Checkout code diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index e4c63a69..e7a990bf 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -7,7 +7,7 @@ on: pull_request: jobs: - format-check: + python-lint: runs-on: ubuntu-latest steps: - name: Checkout code From f3b68628615ad23a00328ef0e015d188ba11d4c4 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:18:29 -0800 Subject: [PATCH 05/28] consolidate boilerplate --- .github/workflows/python_autofix.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/python_autofix.yml b/.github/workflows/python_autofix.yml index 2d4a9e15..96dfc958 100644 --- a/.github/workflows/python_autofix.yml +++ b/.github/workflows/python_autofix.yml @@ -9,18 +9,16 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" - name: Set up Python uses: actions/setup-python@v5 with: python-version: 3.9 cache: 'poetry' - - name: Set up Poetry - uses: Gr1N/setup-poetry@v8 - with: - poetry-version: "1.7.1" - - name: Install dependencies run: poetry install From 8ad1406f419710c40d99736d61bf5238cc887bfb Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:26:42 -0800 Subject: [PATCH 06/28] add slash command --- .../{python_autofix.yml => autofix.yml} | 6 ++++-- .github/workflows/python_format_checks.yml | 2 +- .github/workflows/python_lint.yml | 2 +- .github/workflows/slash_command_dispatch.yml | 16 ++++++++++++++++ 4 files changed, 22 insertions(+), 4 deletions(-) rename .github/workflows/{python_autofix.yml => autofix.yml} (87%) create mode 100644 .github/workflows/slash_command_dispatch.yml diff --git a/.github/workflows/python_autofix.yml b/.github/workflows/autofix.yml similarity index 87% rename from .github/workflows/python_autofix.yml rename to .github/workflows/autofix.yml index 96dfc958..f513c27b 100644 --- a/.github/workflows/python_autofix.yml +++ b/.github/workflows/autofix.yml @@ -1,7 +1,9 @@ -name: Auto-fix Python +name: Auto-fix when '/autofix' Slash Command is used on: - workflow_dispatch: + workflow_dispatch: {} + repository_dispatch: + types: [autofix-command] jobs: python-autofix: diff --git a/.github/workflows/python_format_checks.yml b/.github/workflows/python_format_checks.yml index 05331e05..e3929c59 100644 --- a/.github/workflows/python_format_checks.yml +++ b/.github/workflows/python_format_checks.yml @@ -4,7 +4,7 @@ on: push: branches: - main - pull_request: + pull_request: {} jobs: python-format-check: diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index e7a990bf..3f8ec6bb 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -4,7 +4,7 @@ on: push: branches: - main - pull_request: + pull_request: {} jobs: python-lint: diff --git a/.github/workflows/slash_command_dispatch.yml b/.github/workflows/slash_command_dispatch.yml new file mode 100644 index 00000000..1da8ec92 --- /dev/null +++ b/.github/workflows/slash_command_dispatch.yml @@ -0,0 +1,16 @@ +name: Slash Command Dispatch + +on: + issue_comment: + types: [created] + +jobs: + slashCommandDispatch: + runs-on: ubuntu-latest + steps: + - name: Slash Command Dispatch + uses: peter-evans/slash-command-dispatch@v4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commands: | + autofix From 2ce7a129bfe35b70c6b30dba4f17bce448518934 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:30:54 -0800 Subject: [PATCH 07/28] add pytest --- .github/workflows/python_pytest.yml | 29 ++++++++++++++++++++++++ .github/workflows/python_pytest_fast.yml | 29 ++++++++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 .github/workflows/python_pytest.yml create mode 100644 .github/workflows/python_pytest_fast.yml diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml new file mode 100644 index 00000000..bdb36466 --- /dev/null +++ b/.github/workflows/python_pytest.yml @@ -0,0 +1,29 @@ +name: Run Pytest + +on: + push: + branches: + - main + pull_request: {} + +jobs: + python-lint: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + cache: 'poetry' + + - name: Install dependencies + run: poetry install + + - name: Run Pytest + run: poetry run pytest diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml new file mode 100644 index 00000000..62d36866 --- /dev/null +++ b/.github/workflows/python_pytest_fast.yml @@ -0,0 +1,29 @@ +name: Run Pytest (Fast-Fail) + +on: + push: + branches: + - main + pull_request: {} + +jobs: + python-lint: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Set up Poetry + uses: Gr1N/setup-poetry@v8 + with: + poetry-version: "1.7.1" + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: 3.9 + cache: 'poetry' + + - name: Install dependencies + run: poetry install + + - name: Run Pytest + run: poetry run pytest -m "not slow" --fast-fail From 1afc12d0ea994cd14214a09a613eb881f8f72599 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:33:36 -0800 Subject: [PATCH 08/28] fix job names --- .github/workflows/python_pytest.yml | 2 +- .github/workflows/python_pytest_fast.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index bdb36466..8b021324 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -7,7 +7,7 @@ on: pull_request: {} jobs: - python-lint: + pytest: runs-on: ubuntu-latest steps: - name: Checkout code diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index 62d36866..2440ef50 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -7,7 +7,7 @@ on: pull_request: {} jobs: - python-lint: + pytest-fast: runs-on: ubuntu-latest steps: - name: Checkout code From ff8f35ee3aa7e09899cebc08ca6d96f5d6ca80ab Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:36:30 -0800 Subject: [PATCH 09/28] rename workflows --- .github/workflows/python_format_checks.yml | 2 +- .github/workflows/python_lint.yml | 2 +- .github/workflows/python_pytest.yml | 2 +- .github/workflows/python_pytest_fast.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python_format_checks.yml b/.github/workflows/python_format_checks.yml index e3929c59..2ba2a9b6 100644 --- a/.github/workflows/python_format_checks.yml +++ b/.github/workflows/python_format_checks.yml @@ -1,4 +1,4 @@ -name: Check Python Format +name: Python Format Check on: push: diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index 3f8ec6bb..f9a76658 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -1,4 +1,4 @@ -name: Lint Python +name: Python Lint on: push: diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index 8b021324..24155bd7 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -1,4 +1,4 @@ -name: Run Pytest +name: Pytest on: push: diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index 2440ef50..f18e3adf 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -1,4 +1,4 @@ -name: Run Pytest (Fast-Fail) +name: Pytest (Fast-Fail) on: push: From 1ba0f47483dc93ba913746005c4498bb76bbb856 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:39:31 -0800 Subject: [PATCH 10/28] fix docs --- docs/generated/airbyte.html | 284 ++++++++++++++++++------------------ 1 file changed, 142 insertions(+), 142 deletions(-) diff --git a/docs/generated/airbyte.html b/docs/generated/airbyte.html index 2c5ba3eb..5ad45f6e 100644 --- a/docs/generated/airbyte.html +++ b/docs/generated/airbyte.html @@ -2,14 +2,14 @@
- + class CachedDataset(airbyte.datasets._sql.SQLDataset): - +
- +

A dataset backed by a SQL table cache.

Because this dataset includes all records from the underlying table, we also expose the @@ -19,14 +19,14 @@

- + CachedDataset(cache: 'SQLCacheBase', stream_name: str) - +
- - + +
@@ -36,10 +36,10 @@ def to_pandas(self) -> pandas.core.frame.DataFrame: - +
- +

Return a pandas DataFrame representation of the dataset.

The base implementation simply passes the record iterator to Panda's DataFrame constructor.

@@ -49,15 +49,15 @@
- + def to_sql_table(self) -> 'Table': - +
- - + +
@@ -73,14 +73,14 @@
Inherited Members
- + class DuckDBCache(airbyte.caches.duckdb.DuckDBCacheBase): - +
- +

A DuckDB implementation of the cache.

Parquet is used for local file storage before bulk loading. @@ -94,11 +94,11 @@

Inherited Members
file_writer_class = <class 'airbyte._file_writers.parquet.ParquetWriter'> - +
- - + +
@@ -143,14 +143,14 @@
Inherited Members
- + class DuckDBCacheConfig(airbyte.caches.base.SQLCacheConfigBase, airbyte._file_writers.parquet.ParquetWriterConfig): - +
- +

Configuration for the DuckDB cache.

Also inherits config from the ParquetWriter, which is responsible for writing files to disk.

@@ -161,10 +161,10 @@
Inherited Members
db_path: pathlib.Path | str - +
- +

Normally db_path is a Path object.

There are some cases, such as when connecting to MotherDuck, where it could be a string that @@ -177,10 +177,10 @@

Inherited Members
schema_name: str - +
- +

The name of the schema to write to. Defaults to "main".

@@ -193,10 +193,10 @@
Inherited Members
def get_sql_alchemy_url(self) -> str: - +
- +

Return the SQLAlchemy URL to use.

@@ -204,14 +204,14 @@
Inherited Members
- + def get_database_name(self) -> str: - +
- +

Return the name of the database.

@@ -252,14 +252,14 @@
Inherited Members
- + def get_available_connectors() -> list[str]: - +
- +

Return a list of all available connectors.

Connectors will be returned in alphabetical order, with the standard prefix "source-".

@@ -269,14 +269,14 @@
Inherited Members
- + def get_source( name: str, config: dict[str, typing.Any] | None = None, *, version: str | None = None, pip_url: str | None = None, local_executable: pathlib.Path | str | None = None, install_if_missing: bool = True) -> Source: - +
- +

Get a connector by name and version.

Args: @@ -299,14 +299,14 @@

Inherited Members
- + def get_default_cache() -> DuckDBCache: - +
- +

Get a local cache for storing data, using the default database path.

Cache files are stored in the .cache directory, relative to the current @@ -317,14 +317,14 @@

Inherited Members
- + def get_secret( secret_name: str, source: SecretSource | list[SecretSource] = <SecretSource.ANY: 4>, *, prompt: bool = True) -> str: - +
- +

Get a secret from the environment.

The optional source argument of enum type SecretSource or list of SecretSource options. @@ -340,14 +340,14 @@

Inherited Members
- + def new_local_cache( cache_name: str | None = None, cache_dir: str | pathlib.Path | None = None, *, cleanup: bool = True) -> DuckDBCache: - +
- +

Get a local cache for storing data, using a name string to seed the path.

Args: @@ -363,14 +363,14 @@

Inherited Members
- + class ReadResult(collections.abc.Mapping[str, airbyte.datasets._sql.CachedDataset]): - +
- +

A Mapping is a generic container for associating key/value pairs.

@@ -381,60 +381,60 @@
Inherited Members
- + ReadResult( processed_records: int, cache: airbyte.caches.base.SQLCacheBase, processed_streams: list[str]) - +
- - + +
processed_records - +
- - + +
- + def get_sql_engine(self) -> sqlalchemy.engine.base.Engine: - +
- - + +
streams: collections.abc.Mapping[str, CachedDataset] - +
- - + +
cache: airbyte.caches.base.SQLCacheBase - +
- - + +
@@ -452,14 +452,14 @@
Inherited Members
- + class SecretSource(enum.Enum): - +
- +

An enumeration.

@@ -469,11 +469,11 @@
Inherited Members
ENV = <SecretSource.ENV: 1> - + - - + +
@@ -481,11 +481,11 @@
Inherited Members
DOTENV = <SecretSource.DOTENV: 2> - +
- - + +
@@ -493,11 +493,11 @@
Inherited Members
GOOGLE_COLAB = <SecretSource.GOOGLE_COLAB: 3> - +
- - + +
@@ -505,11 +505,11 @@
Inherited Members
ANY = <SecretSource.ANY: 4> - +
- - + +
@@ -517,11 +517,11 @@
Inherited Members
PROMPT = <SecretSource.PROMPT: 5> - +
- - + +
@@ -537,27 +537,27 @@
Inherited Members
- + class Source: - +
- +

A class representing a source that can be called.

- + Source( executor: airbyte._executor.Executor, name: str, config: dict[str, typing.Any] | None = None, streams: list[str] | None = None, *, validate: bool = False) - +
- +

Initialize the source.

If config is provided, it will be validated against the spec if validate is True.

@@ -569,34 +569,34 @@
Inherited Members
executor - +
- - + +
name - +
- - + +
- + def set_streams(self, streams: list[str]) -> None: - +
- +

Deprecated. See select_streams().

@@ -604,14 +604,14 @@
Inherited Members
- + def select_all_streams(self) -> None: - +
- +

Select all streams.

This is a more streamlined equivalent to:

@@ -625,14 +625,14 @@
Inherited Members
- + def select_streams(self, streams: list[str]) -> None: - +
- +

Select the stream names that should be read from the connector.

Currently, if this is not set, all streams will be read.

@@ -642,14 +642,14 @@
Inherited Members
- + def get_selected_streams(self) -> list[str]: - +
- +

Get the selected streams.

If no streams are selected, return an empty list.

@@ -659,14 +659,14 @@
Inherited Members
- + def set_config(self, config: dict[str, typing.Any], *, validate: bool = False) -> None: - +
- +

Set the config for the connector.

If validate is True, raise an exception if the config fails validation.

@@ -679,14 +679,14 @@
Inherited Members
- + def get_config(self) -> dict[str, typing.Any]: - +
- +

Get the config for the connector.

@@ -694,14 +694,14 @@
Inherited Members
- + def validate_config(self, config: dict[str, typing.Any] | None = None) -> None: - +
- +

Validate the config against the spec.

If config is not provided, the already-set config will be validated.

@@ -711,14 +711,14 @@
Inherited Members
- + def get_available_streams(self) -> list[str]: - +
- +

Get the available streams from the spec.

@@ -728,10 +728,10 @@
Inherited Members
docs_url: str - +
- +

Get the URL to the connector's documentation.

@@ -741,10 +741,10 @@
Inherited Members
discovered_catalog: airbyte_protocol.models.airbyte_protocol.AirbyteCatalog - +
- +

Get the raw catalog for the given streams.

If the catalog is not yet known, we call discover to get it.

@@ -756,10 +756,10 @@
Inherited Members
configured_catalog: airbyte_protocol.models.airbyte_protocol.ConfiguredAirbyteCatalog - +
- +

Get the configured catalog for the given streams.

If the raw catalog is not yet known, we call discover to get it.

@@ -775,14 +775,14 @@
Inherited Members
- + def get_records(self, stream: str) -> airbyte.datasets._lazy.LazyDataset: - +
- +

Read a stream from the connector.

This involves the following steps:

@@ -801,14 +801,14 @@
Inherited Members
- + def check(self) -> None: - +
- +

Call check on the connector.

This involves the following steps:

@@ -825,14 +825,14 @@
Inherited Members
- + def install(self) -> None: - +
- +

Install the connector if it is not yet installed.

@@ -840,14 +840,14 @@
Inherited Members
- + def uninstall(self) -> None: - +
- +

Uninstall the connector if it is installed.

This only works if the use_local_install flag wasn't used and installation is managed by @@ -858,14 +858,14 @@

Inherited Members
- + def read( self, cache: airbyte.caches.base.SQLCacheBase | None = None, *, write_strategy: str | airbyte.strategies.WriteStrategy = <WriteStrategy.AUTO: 'auto'>, force_full_refresh: bool = False) -> ReadResult: - +
- +

Read from the connector and write to the cache.

Args: @@ -886,4 +886,4 @@

Inherited Members
- + \ No newline at end of file From 9e46090afac51e4545bdbbc2275fc3a089aa249c Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:50:42 -0800 Subject: [PATCH 11/28] sort dependencies --- poetry.lock | 39 ++++++++++++++------------------------- pyproject.toml | 25 ++++++++++++------------- 2 files changed, 26 insertions(+), 38 deletions(-) diff --git a/poetry.lock b/poetry.lock index 86d14143..9819e186 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "airbyte-cdk" @@ -1413,22 +1413,22 @@ testing = ["google-api-core[grpc] (>=1.31.5)"] [[package]] name = "protobuf" -version = "4.25.2" +version = "4.25.3" description = "" optional = false python-versions = ">=3.8" files = [ - {file = "protobuf-4.25.2-cp310-abi3-win32.whl", hash = "sha256:b50c949608682b12efb0b2717f53256f03636af5f60ac0c1d900df6213910fd6"}, - {file = "protobuf-4.25.2-cp310-abi3-win_amd64.whl", hash = "sha256:8f62574857ee1de9f770baf04dde4165e30b15ad97ba03ceac65f760ff018ac9"}, - {file = "protobuf-4.25.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:2db9f8fa64fbdcdc93767d3cf81e0f2aef176284071507e3ede160811502fd3d"}, - {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:10894a2885b7175d3984f2be8d9850712c57d5e7587a2410720af8be56cdaf62"}, - {file = "protobuf-4.25.2-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:fc381d1dd0516343f1440019cedf08a7405f791cd49eef4ae1ea06520bc1c020"}, - {file = "protobuf-4.25.2-cp38-cp38-win32.whl", hash = "sha256:33a1aeef4b1927431d1be780e87b641e322b88d654203a9e9d93f218ee359e61"}, - {file = "protobuf-4.25.2-cp38-cp38-win_amd64.whl", hash = "sha256:47f3de503fe7c1245f6f03bea7e8d3ec11c6c4a2ea9ef910e3221c8a15516d62"}, - {file = "protobuf-4.25.2-cp39-cp39-win32.whl", hash = "sha256:5e5c933b4c30a988b52e0b7c02641760a5ba046edc5e43d3b94a74c9fc57c1b3"}, - {file = "protobuf-4.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:d66a769b8d687df9024f2985d5137a337f957a0916cf5464d1513eee96a63ff0"}, - {file = "protobuf-4.25.2-py3-none-any.whl", hash = "sha256:a8b7a98d4ce823303145bf3c1a8bdb0f2f4642a414b196f04ad9853ed0c8f830"}, - {file = "protobuf-4.25.2.tar.gz", hash = "sha256:fe599e175cb347efc8ee524bcd4b902d11f7262c0e569ececcb89995c15f0a5e"}, + {file = "protobuf-4.25.3-cp310-abi3-win32.whl", hash = "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa"}, + {file = "protobuf-4.25.3-cp310-abi3-win_amd64.whl", hash = "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8"}, + {file = "protobuf-4.25.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c"}, + {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019"}, + {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d"}, + {file = "protobuf-4.25.3-cp38-cp38-win32.whl", hash = "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"}, + {file = "protobuf-4.25.3-cp38-cp38-win_amd64.whl", hash = "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4"}, + {file = "protobuf-4.25.3-cp39-cp39-win32.whl", hash = "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4"}, + {file = "protobuf-4.25.3-cp39-cp39-win_amd64.whl", hash = "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c"}, + {file = "protobuf-4.25.3-py3-none-any.whl", hash = "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9"}, + {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"}, ] [[package]] @@ -1924,7 +1924,6 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, - {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -1932,16 +1931,8 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, - {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, - {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef"}, - {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, - {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, - {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, - {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -1958,7 +1949,6 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, - {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -1966,7 +1956,6 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, - {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2684,4 +2673,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "3b46151e994684f0953be1041a850ac8efcedea10632f8fa86aaaa6d20385174" +content-hash = "a61f9755ed5b078d77d06d6930ddbc394a88e480e41ad872547fb542aeb7ec0c" diff --git a/pyproject.toml b/pyproject.toml index ddaa8c37..f8defa35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,27 +10,26 @@ packages = [{include = "airbyte"}] python = "^3.9" airbyte-cdk = "^0.58.3" -# airbyte-protocol-models = "^1.0.1" # Conflicts with airbyte-cdk # TODO: delete or resolve +duckdb-engine = "^0.10.0" jsonschema = "3.2.0" orjson = "^3.9.10" overrides = "^7.4.0" pandas = "2.1.4" # 2.2.0 breaks sqlalchemy interop - TODO: optionally retest higher versions +pendulum = "<=3.0.0" psycopg2-binary = "^2.9.9" +# psycopg = {extras = ["binary", "pool"], version = "^3.1.16"} +# Psycopg3 is not supported in SQLAlchemy 1.x: +pyarrow = "^14.0.2" +pydantic = "<=2.0" +python-dotenv = "^1.0.1" python-ulid = "^2.2.0" -types-pyyaml = "^6.0.12.12" -ulid = "^1.1" -sqlalchemy = "1.4.51" -snowflake-connector-python = "3.6.0" -snowflake-sqlalchemy = "^1.5.1" -duckdb-engine = "^0.10.0" requests = "^2.31.0" -pyarrow = "^14.0.2" - -# Psycopg3 is not supported in SQLAlchemy 1.x: -# psycopg = {extras = ["binary", "pool"], version = "^3.1.16"} rich = "^13.7.0" -pendulum = "<=3.0.0" -python-dotenv = "^1.0.1" +snowflake-connector-python = "3.6.0" +snowflake-sqlalchemy = "^1.5.1" +sqlalchemy = "1.4.51" +types-pyyaml = "^6.0.12.12" +ulid = "^1.1" [tool.poetry.group.dev.dependencies] From efa1ac650c613a8ad353609bf8ff339a51c24557 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:52:44 -0800 Subject: [PATCH 12/28] clean up --- .github/workflows/python_format_checks.yml | 2 +- .github/workflows/python_lint.yml | 2 +- .github/workflows/python_pytest.yml | 2 +- .github/workflows/python_pytest_fast.yml | 2 +- airbyte/caches/base.py | 1 - 5 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/python_format_checks.yml b/.github/workflows/python_format_checks.yml index 2ba2a9b6..9e4d2824 100644 --- a/.github/workflows/python_format_checks.yml +++ b/.github/workflows/python_format_checks.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10 cache: 'poetry' - name: Install dependencies diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index f9a76658..90f01288 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10 cache: 'poetry' - name: Install dependencies diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index 24155bd7..6b88ae98 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10 cache: 'poetry' - name: Install dependencies diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index f18e3adf..4b32ef97 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: 3.10 cache: 'poetry' - name: Install dependencies diff --git a/airbyte/caches/base.py b/airbyte/caches/base.py index 0d44daa4..02a1e977 100644 --- a/airbyte/caches/base.py +++ b/airbyte/caches/base.py @@ -1,5 +1,4 @@ # Copyright (c) 2023 Airbyte, Inc., all rights reserved. - """A SQL Cache implementation.""" from __future__ import annotations From 9401b2161123471166ae4a5224347735bfa6f1dd Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:55:36 -0800 Subject: [PATCH 13/28] fix python version, add pytest python version matrix --- .github/workflows/python_format_checks.yml | 2 +- .github/workflows/python_lint.yml | 2 +- .github/workflows/python_pytest.yml | 5 ++++- .github/workflows/python_pytest_fast.yml | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/python_format_checks.yml b/.github/workflows/python_format_checks.yml index 9e4d2824..834f2fa1 100644 --- a/.github/workflows/python_format_checks.yml +++ b/.github/workflows/python_format_checks.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' cache: 'poetry' - name: Install dependencies diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index 90f01288..686e9082 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' cache: 'poetry' - name: Install dependencies diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index 6b88ae98..eb164db0 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -9,6 +9,9 @@ on: jobs: pytest: runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.10', '3.11'] steps: - name: Checkout code uses: actions/checkout@v4 @@ -19,7 +22,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: ${{ matrix.python-version }} cache: 'poetry' - name: Install dependencies diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index 4b32ef97..83e1e98f 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -19,7 +19,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: 3.10 + python-version: '3.10' cache: 'poetry' - name: Install dependencies From 805c6f47221d8d88d1d9ccd911c91f7a28d68012 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 21:58:48 -0800 Subject: [PATCH 14/28] fix arg --- .github/workflows/python_pytest_fast.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index 83e1e98f..8ddc7ae9 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -26,4 +26,4 @@ jobs: run: poetry install - name: Run Pytest - run: poetry run pytest -m "not slow" --fast-fail + run: poetry run pytest -m "not slow" --exitfirst From f43cc608ce3e1bbb0e9369754845aee01b3fed1a Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 22:03:43 -0800 Subject: [PATCH 15/28] change name --- .github/workflows/python_pytest_fast.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index 8ddc7ae9..848ad077 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -1,4 +1,4 @@ -name: Pytest (Fast-Fail) +name: Pytest (Fail-Fast) on: push: From 65bc616da122541b9a09859fd3682629a7c0b0d2 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 22:20:17 -0800 Subject: [PATCH 16/28] reference secrets.GCP_GSM_CREDENTIALS --- .github/workflows/python_pytest.yml | 2 ++ .github/workflows/python_pytest_fast.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index eb164db0..ab71c025 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -29,4 +29,6 @@ jobs: run: poetry install - name: Run Pytest + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} run: poetry run pytest diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index 848ad077..4a9d6c5d 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -26,4 +26,6 @@ jobs: run: poetry install - name: Run Pytest + env: + GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} run: poetry run pytest -m "not slow" --exitfirst From 44fd6499b352cc5b61955f781b8c597c26cc9287 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 22:23:23 -0800 Subject: [PATCH 17/28] rename failfast --- .github/workflows/python_pytest_fast.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index 4a9d6c5d..f0ed3d13 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -1,4 +1,4 @@ -name: Pytest (Fail-Fast) +name: Pytest (Fast Tests Only) on: push: From f945ca48655bc2226ba188c32ac1ce6a4eb3cf68 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 22:27:01 -0800 Subject: [PATCH 18/28] rename pytest (all) --- .github/workflows/python_pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index ab71c025..86f0685b 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -1,4 +1,4 @@ -name: Pytest +name: Pytest (all) on: push: From f9532814f8181adde3ef4e432a39f5cab4a9cf5c Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 22:27:25 -0800 Subject: [PATCH 19/28] match casing --- .github/workflows/python_pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index 86f0685b..02f9c076 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -1,4 +1,4 @@ -name: Pytest (all) +name: Pytest (All) on: push: From a8a479238d4a5c1ece631c1f2f90a7e272c53625 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 22:28:19 -0800 Subject: [PATCH 20/28] rename lint job --- .github/workflows/python_lint.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_lint.yml b/.github/workflows/python_lint.yml index 686e9082..bcc8d337 100644 --- a/.github/workflows/python_lint.yml +++ b/.github/workflows/python_lint.yml @@ -1,4 +1,4 @@ -name: Python Lint +name: Python Lint Check on: push: From 9f5c07d5f3e67c3f46857bf652ceac80d44e80d3 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 22:42:21 -0800 Subject: [PATCH 21/28] skip creds with failfast tests --- .github/workflows/python_pytest_fast.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index f0ed3d13..286d1525 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -28,4 +28,4 @@ jobs: - name: Run Pytest env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} - run: poetry run pytest -m "not slow" --exitfirst + run: poetry run pytest -m "not slow and not requires_creds" --exitfirst From ff3368beaf7e3bc0e813a6f312f8fb36125af900 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 22:42:53 -0800 Subject: [PATCH 22/28] skip tests that require creds when running on forks --- .github/workflows/python_pytest.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index 02f9c076..d47f5a8b 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -8,10 +8,14 @@ on: jobs: pytest: + # Don't run on forks + if: github.repository_owner == 'airbytehq' + runs-on: ubuntu-latest strategy: matrix: python-version: ['3.10', '3.11'] + steps: - name: Checkout code uses: actions/checkout@v4 From e543ede059f257e5d3e774cf06b0c53788bc3259 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 22:56:26 -0800 Subject: [PATCH 23/28] add tests for py3.9 --- .github/workflows/python_pytest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index d47f5a8b..f1d86d05 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.10', '3.11'] + python-version: ['3.9', '3.10', '3.11'] steps: - name: Checkout code From 06fac0a3e1e9384efd52cabd94d332e342c39511 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 23:03:00 -0800 Subject: [PATCH 24/28] try rename --- .github/workflows/python_pytest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index f1d86d05..d6c71f96 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -8,6 +8,7 @@ on: jobs: pytest: + name: Pytest (All, Python ${{ matrix.python-version }}) # Don't run on forks if: github.repository_owner == 'airbytehq' From 937fe60fe1585629287891f091b783d331e1dc8b Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 23:05:30 -0800 Subject: [PATCH 25/28] don't failfast --- .github/workflows/python_pytest.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index d6c71f96..ac3f8399 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -16,6 +16,7 @@ jobs: strategy: matrix: python-version: ['3.9', '3.10', '3.11'] + fail-fast: false steps: - name: Checkout code From b968690c6db653a5d525bfcf5d06dcd94bdc1dea Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 23:09:22 -0800 Subject: [PATCH 26/28] add comments describing workflows --- .github/workflows/python_pytest.yml | 2 ++ .github/workflows/python_pytest_fast.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index ac3f8399..55cb2a12 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -1,3 +1,5 @@ +# This workflow will run pytest, prioritizing rebustness over speed. +# This is in contrast to the 'failfast' workflow, which prioritizes speed over robustness. name: Pytest (All) on: diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index 286d1525..f06cd68c 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -1,3 +1,5 @@ +# This workflow will run pytest, prioritizing speed over robustness. +# This is in contrast to the 'all' workflow, which prioritizes robustness over speed. name: Pytest (Fast Tests Only) on: From d97b8bf61571f61e4d4b84671901777fd7424f39 Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 23:13:33 -0800 Subject: [PATCH 27/28] workaround: python tests on multi-version --- .github/workflows/python_pytest.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/python_pytest.yml b/.github/workflows/python_pytest.yml index 55cb2a12..99af6997 100644 --- a/.github/workflows/python_pytest.yml +++ b/.github/workflows/python_pytest.yml @@ -17,7 +17,12 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: ['3.9', '3.10', '3.11'] + python-version: [ + # TODO: Re-enable 3.9 and 3.11 once we have stable tests across all versions. + # '3.9', + '3.10', + # '3.11', + ] fail-fast: false steps: From 0cf9b6ade0852340fba38e72a802a67e2b48ca1e Mon Sep 17 00:00:00 2001 From: Aaron Steers Date: Thu, 15 Feb 2024 23:15:12 -0800 Subject: [PATCH 28/28] update description --- .github/workflows/python_pytest_fast.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python_pytest_fast.yml b/.github/workflows/python_pytest_fast.yml index f06cd68c..1a829f59 100644 --- a/.github/workflows/python_pytest_fast.yml +++ b/.github/workflows/python_pytest_fast.yml @@ -27,7 +27,7 @@ jobs: - name: Install dependencies run: poetry install - - name: Run Pytest + - name: Run Pytest (Fast Tests Only) env: GCP_GSM_CREDENTIALS: ${{ secrets.GCP_GSM_CREDENTIALS }} run: poetry run pytest -m "not slow and not requires_creds" --exitfirst