From 740deeb8726ac7fd6d08ef22f7ffcc621f1efe23 Mon Sep 17 00:00:00 2001 From: "Aaron (\"AJ\") Steers" Date: Fri, 12 Apr 2024 08:25:57 -0700 Subject: [PATCH] Docs: Add code samples/snippets; Feat: add `parse_json()` as convenience method on Handles (#182) --- airbyte/cloud/__init__.py | 35 ++++++++++++++++++++++++---- airbyte/secrets/base.py | 12 +++++++++- airbyte/secrets/google_gsm.py | 44 ++++++++++++++--------------------- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/airbyte/cloud/__init__.py b/airbyte/cloud/__init__.py index cd58e5de..512da317 100644 --- a/airbyte/cloud/__init__.py +++ b/airbyte/cloud/__init__.py @@ -3,23 +3,50 @@ You can use this module to interact with Airbyte Cloud, OSS, and Enterprise. -Usage example: +## Examples + +### Basic Usage Example: ```python import airbyte as ab from airbyte import cloud +# Initialize an Airbyte Cloud workspace object workspace = cloud.CloudWorkspace( workspace_id="123", api_key=ab.get_secret("AIRBYTE_CLOUD_API_KEY"), ) -sync_result = workspace.run_sync( - connection_id="456", -) +# Run a sync job on Airbyte Cloud +connection = workspace.get_connection(connection_id="456") +sync_result = connection.run_sync() print(sync_result.get_job_status()) ``` +### Example Read From Cloud Destination: + +If your destination is supported, you can read records directly from the +`SyncResult` object. Currently this is supported in Snowflake and BigQuery only. + + +``` +# Assuming we've already created a `connection` object + +# Get the latest job result and print the stream names +sync_result = connection.get_sync_result() +print(sync_result.stream_names) + +# Get a dataset from the sync result +dataset: CachedDataset = sync_result.get_dataset("users") + +# Get a SQLAlchemy table to use in SQL queries... +users_table = dataset.to_sql_table() +print(f"Table name: {users_table.name}") + +# Or iterate over the dataset directly +for record in dataset: + print(record) +``` ℹ️ **Experimental Features** diff --git a/airbyte/secrets/base.py b/airbyte/secrets/base.py index d77ac633..3ae6b7fe 100644 --- a/airbyte/secrets/base.py +++ b/airbyte/secrets/base.py @@ -126,7 +126,8 @@ class SecretHandle: """A handle for a secret in a secret manager. This class is used to store a reference to a secret in a secret manager. - The secret is not retrieved until the `get_value()` method is called on the handle. + The secret is not retrieved until the `get_value()` or `parse_json()` methods are + called. """ def __init__( @@ -144,3 +145,12 @@ def get_value(self) -> SecretString: Subclasses can optionally override this method to provide a more optimized code path. """ return cast(SecretString, self.parent.get_secret(self.secret_name)) + + def parse_json(self) -> dict: + """Parse the secret as JSON. + + This method is a convenience method to parse the secret as JSON without + needing to call `get_value()` first. If the secret is not a valid JSON + string, a `PyAirbyteInputError` will be raised. + """ + return self.get_value().parse_json() diff --git a/airbyte/secrets/google_gsm.py b/airbyte/secrets/google_gsm.py index c23c19e3..b361dbdb 100644 --- a/airbyte/secrets/google_gsm.py +++ b/airbyte/secrets/google_gsm.py @@ -4,39 +4,29 @@ Usage Example: ```python -gsm_secrets_manager = GoogleGSMSecretManager( - project=AIRBYTE_INTERNAL_GCP_PROJECT, +import ab +from airbyte import secrets + +# Initialize the Google GSM secret manager +gsm = secrets.GoogleGSMSecretManager( + project="my-project", credentials_json=ab.get_secret("GCP_GSM_CREDENTIALS"), ) -first_secret: SecretHandle = next( - gsm_secrets_manager.fetch_connector_secrets( - connector_name=connector_name, - ), - None, -) +# Get secrets with the label 'connector=source-github' +secrets = gsm.fetch_connector_secrets("source-github") -print(f"Found '{connector_name}' credential secret '${first_secret.secret_name}'.") -return first_secret.get_value().parse_json() -``` +# Get the first secret and parse the JSON value +config: dict = next(secrets, None).parse_json() -More compact example: - -```python -gsm_secrets_manager = GoogleGSMSecretManager( - project=AIRBYTE_INTERNAL_GCP_PROJECT, - credentials_json=ab.get_secret("GCP_GSM_CREDENTIALS"), -) -connector_config: dict = ( - next( - gsm_secrets_manager.fetch_connector_secrets( - connector_name=connector_name, - ), - None, - ) - .get_value() - .parse_json() +# Pass the config to your source +source = ab.get_source( + "source-github", + config=config, + streams="*", ) +read_result = source.read() ``` + """ from __future__ import annotations