From 83cda85b600a257094ca204fe15f55f0d9d82ad8 Mon Sep 17 00:00:00 2001 From: Krishan Patel-Watts Date: Wed, 1 May 2024 09:24:07 +0100 Subject: [PATCH] add custom assembled connector --- .../source-assembled-custom/README.md | 103 ++++ .../source-assembled-custom/__init__.py | 3 + .../acceptance-test-config.yml | 37 ++ .../integration_tests/__init__.py | 3 + .../integration_tests/abnormal_state.json | 5 + .../integration_tests/acceptance.py | 16 + .../integration_tests/configured_catalog.json | 22 + .../integration_tests/invalid_config.json | 3 + .../integration_tests/sample_config.json | 3 + .../integration_tests/sample_state.json | 5 + .../source-assembled-custom/main.py | 13 + .../source-assembled-custom/metadata.yaml | 35 ++ .../source-assembled-custom/pyproject.toml | 27 + .../source_assembled_custom/__init__.py | 8 + .../source_assembled_custom/manifest.yaml | 497 ++++++++++++++++++ .../source_assembled_custom/run.py | 13 + .../source_assembled_custom/schemas/TODO.md | 16 + .../schemas/customers.json | 16 + .../schemas/employees.json | 19 + .../source_assembled_custom/source.py | 18 + 20 files changed, 862 insertions(+) create mode 100644 airbyte-integrations/connectors/source-assembled-custom/README.md create mode 100644 airbyte-integrations/connectors/source-assembled-custom/__init__.py create mode 100644 airbyte-integrations/connectors/source-assembled-custom/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-assembled-custom/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-assembled-custom/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-assembled-custom/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-assembled-custom/integration_tests/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-assembled-custom/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-assembled-custom/integration_tests/sample_config.json create mode 100644 airbyte-integrations/connectors/source-assembled-custom/integration_tests/sample_state.json create mode 100644 airbyte-integrations/connectors/source-assembled-custom/main.py create mode 100644 airbyte-integrations/connectors/source-assembled-custom/metadata.yaml create mode 100644 airbyte-integrations/connectors/source-assembled-custom/pyproject.toml create mode 100644 airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/__init__.py create mode 100644 airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/manifest.yaml create mode 100644 airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/run.py create mode 100644 airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/TODO.md create mode 100644 airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/customers.json create mode 100644 airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/employees.json create mode 100644 airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/source.py diff --git a/airbyte-integrations/connectors/source-assembled-custom/README.md b/airbyte-integrations/connectors/source-assembled-custom/README.md new file mode 100644 index 000000000000..e28e133e8b7e --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/README.md @@ -0,0 +1,103 @@ +# Assembled Custom Source + +This is the repository for the Assembled Custom configuration based source connector. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.com/integrations/sources/assembled-custom). + +## Local development + +### Prerequisites + +* Python (`^3.9`) +* Poetry (`^1.7`) - installation instructions [here](https://python-poetry.org/docs/#installation) + + + +### Installing the connector + +From this connector directory, run: +```bash +poetry install --with dev +``` + + +### Create credentials + +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.com/integrations/sources/assembled-custom) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `src/source_assembled_custom/spec.yaml` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `sample_files/sample_config.json` for a sample config file. + + +### Locally running the connector + +``` +poetry run source-assembled-custom spec +poetry run source-assembled-custom check --config secrets/config.json +poetry run source-assembled-custom discover --config secrets/config.json +poetry run source-assembled-custom read --config secrets/config.json --catalog sample_files/configured_catalog.json +``` + +### Running tests + +To run tests locally, from the connector directory run: + +``` +poetry run pytest tests +``` + +### Building the docker image + +1. Install [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md) +2. Run the following command to build the docker image: +```bash +airbyte-ci connectors --name=source-assembled-custom build +``` + +An image will be available on your host with the tag `airbyte/source-assembled-custom:dev`. + + +### Running as a docker container + +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-assembled-custom:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-assembled-custom:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-assembled-custom:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-assembled-custom:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` + +### Running our CI test suite + +You can run our full test suite locally using [`airbyte-ci`](https://github.com/airbytehq/airbyte/blob/master/airbyte-ci/connectors/pipelines/README.md): +```bash +airbyte-ci connectors --name=source-assembled-custom test +``` + +### Customizing acceptance Tests + +Customize `acceptance-test-config.yml` file to configure acceptance tests. See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. + +### Dependency Management + +All of your dependencies should be managed via Poetry. +To add a new dependency, run: +```bash +poetry add +``` + +Please commit the changes to `pyproject.toml` and `poetry.lock` files. + +## Publishing a new version of the connector + +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing our test suite: `airbyte-ci connectors --name=source-assembled-custom test` +2. Bump the connector version (please follow [semantic versioning for connectors](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#semantic-versioning-for-connectors)): + - bump the `dockerImageTag` value in in `metadata.yaml` + - bump the `version` value in `pyproject.toml` +3. Make sure the `metadata.yaml` content is up to date. +4. Make sure the connector documentation and its changelog is up to date (`docs/integrations/sources/assembled-custom.md`). +5. Create a Pull Request: use [our PR naming conventions](https://docs.airbyte.com/contributing-to-airbyte/resources/pull-requests-handbook/#pull-request-title-convention). +6. Pat yourself on the back for being an awesome contributor. +7. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. +8. Once your PR is merged, the new version of the connector will be automatically published to Docker Hub and our connector registry. \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-assembled-custom/__init__.py b/airbyte-integrations/connectors/source-assembled-custom/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-assembled-custom/acceptance-test-config.yml b/airbyte-integrations/connectors/source-assembled-custom/acceptance-test-config.yml new file mode 100644 index 000000000000..88aad226d4ff --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/acceptance-test-config.yml @@ -0,0 +1,37 @@ +# See [Connector Acceptance Tests](https://docs.airbyte.com/connector-development/testing-connectors/connector-acceptance-tests-reference) +# for more information about how to configure these tests +connector_image: airbyte/source-assembled-custom:dev +acceptance_tests: + spec: + tests: + - spec_path: "source_assembled_custom/spec.yaml" + connection: + tests: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + tests: + - config_path: "secrets/config.json" + basic_read: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" + empty_streams: [] +# TODO uncomment this block to specify that the tests should assert the connector outputs the records provided in the input file a file +# expect_records: +# path: "integration_tests/expected_records.jsonl" +# exact_order: no + incremental: + bypass_reason: "This connector does not implement incremental sync" +# TODO uncomment this block this block if your connector implements incremental sync: +# tests: +# - config_path: "secrets/config.json" +# configured_catalog_path: "integration_tests/configured_catalog.json" +# future_state: +# future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + tests: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-assembled-custom/integration_tests/__init__.py b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/__init__.py new file mode 100644 index 000000000000..c941b3045795 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/__init__.py @@ -0,0 +1,3 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# diff --git a/airbyte-integrations/connectors/source-assembled-custom/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..52b0f2c2118f --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/abnormal_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "todo-abnormal-value" + } +} diff --git a/airbyte-integrations/connectors/source-assembled-custom/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/acceptance.py new file mode 100644 index 000000000000..9e6409236281 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/acceptance.py @@ -0,0 +1,16 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("connector_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + # TODO: setup test dependencies if needed. otherwise remove the TODO comments + yield + # TODO: clean up test dependencies diff --git a/airbyte-integrations/connectors/source-assembled-custom/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/configured_catalog.json new file mode 100644 index 000000000000..36f0468db0d8 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/configured_catalog.json @@ -0,0 +1,22 @@ +{ + "streams": [ + { + "stream": { + "name": "customers", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "employees", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "sync_mode": "incremental", + "destination_sync_mode": "append" + } + ] +} diff --git a/airbyte-integrations/connectors/source-assembled-custom/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/invalid_config.json new file mode 100644 index 000000000000..f3732995784f --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/invalid_config.json @@ -0,0 +1,3 @@ +{ + "todo-wrong-field": "this should be an incomplete config file, used in standard tests" +} diff --git a/airbyte-integrations/connectors/source-assembled-custom/integration_tests/sample_config.json b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/sample_config.json new file mode 100644 index 000000000000..ecc4913b84c7 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/sample_config.json @@ -0,0 +1,3 @@ +{ + "fix-me": "TODO" +} diff --git a/airbyte-integrations/connectors/source-assembled-custom/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/sample_state.json new file mode 100644 index 000000000000..3587e579822d --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/integration_tests/sample_state.json @@ -0,0 +1,5 @@ +{ + "todo-stream-name": { + "todo-field-name": "value" + } +} diff --git a/airbyte-integrations/connectors/source-assembled-custom/main.py b/airbyte-integrations/connectors/source-assembled-custom/main.py new file mode 100644 index 000000000000..a65061b9cef0 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_assembled_custom import SourceAssembledCustom + +if __name__ == "__main__": + source = SourceAssembledCustom() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-assembled-custom/metadata.yaml b/airbyte-integrations/connectors/source-assembled-custom/metadata.yaml new file mode 100644 index 000000000000..7891e4b456a8 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/metadata.yaml @@ -0,0 +1,35 @@ +data: + allowedHosts: + hosts: + - TODO # Please change to the hostname of the source. + registries: + oss: + enabled: true + cloud: + enabled: false + remoteRegistries: + pypi: + enabled: true + packageName: airbyte-source-assembled-custom + connectorBuildOptions: + # Please update to the latest version of the connector base image. + # https://hub.docker.com/r/airbyte/python-connector-base + # Please use the full address with sha256 hash to guarantee build reproducibility. + baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 + connectorSubtype: api + connectorType: source + definitionId: 71871a81-be5e-4ade-b62d-a71dabb1d4c4 + dockerImageTag: 0.1.0 + dockerRepository: airbyte/source-assembled-custom + githubIssueLabel: source-assembled-custom + icon: assembled-custom.svg + license: MIT + name: Assembled Custom + releaseDate: TODO + releaseStage: alpha + supportLevel: community + documentationUrl: https://docs.airbyte.com/integrations/sources/assembled-custom + tags: + - language:python + - cdk:low-code +metadataSpecVersion: "1.0" diff --git a/airbyte-integrations/connectors/source-assembled-custom/pyproject.toml b/airbyte-integrations/connectors/source-assembled-custom/pyproject.toml new file mode 100644 index 000000000000..ea06c455cdc3 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = [ "poetry-core>=1.0.0",] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +version = "0.1.0" +name = "source-assembled-custom" +description = "Source implementation for assembled-custom." +authors = [ "Airbyte ",] +license = "MIT" +readme = "README.md" +documentation = "https://docs.airbyte.com/integrations/sources/assembled-custom" +homepage = "https://airbyte.com" +repository = "https://github.com/airbytehq/airbyte" +packages = [ { include = "source_assembled_custom" }, {include = "main.py" } ] + +[tool.poetry.dependencies] +python = "^3.9,<3.12" +airbyte-cdk = "^0" + +[tool.poetry.scripts] +source-assembled-custom = "source_assembled_custom.run:run" + +[tool.poetry.group.dev.dependencies] +requests-mock = "*" +pytest-mock = "*" +pytest = "*" diff --git a/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/__init__.py b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/__init__.py new file mode 100644 index 000000000000..f15aaddf8d12 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/__init__.py @@ -0,0 +1,8 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from .source import SourceAssembledCustom + +__all__ = ["SourceAssembledCustom"] diff --git a/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/manifest.yaml b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/manifest.yaml new file mode 100644 index 000000000000..2553363f6bea --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/manifest.yaml @@ -0,0 +1,497 @@ +spec: + type: Spec + connection_specification: + type: object + $schema: http://json-schema.org/draft-07/schema# + required: + - start_date + - username + properties: + password: + type: string + order: 2 + title: Password + always_show: true + airbyte_secret: true + username: + type: string + order: 1 + title: Username + start_date: + type: string + order: 0 + title: Start date + format: date-time + pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$ + additionalProperties: true +type: DeclarativeSource +check: + type: CheckStream + stream_names: + - people +streams: + - name: people + type: DeclarativeStream + retriever: + type: SimpleRetriever + paginator: + type: NoPagination + requester: + path: people + type: HttpRequester + url_base: https://api.assembledhq.com/v0/ + http_method: GET + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + request_headers: {} + request_body_json: {} + request_parameters: + limit: '500' + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - people + - '*' + partition_router: [] + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + id: + type: string + role: + type: string + site: + type: + - 'null' + - string + email: + type: string + teams: + type: array + items: + type: string + queues: + type: array + items: + type: string + skills: + type: array + deleted: + type: boolean + agent_id: + type: + - 'null' + - string + channels: + type: + - array + - 'null' + items: + type: string + end_date: + type: + - 'null' + - number + timezone: + type: string + last_name: + type: string + platforms: + type: object + properties: + hibob: + type: string + intercom: + type: string + staffable: + type: boolean + agent_role: + type: string + created_at: + type: number + first_name: + type: string + start_date: + type: + - 'null' + - number + imported_id: + type: + - 'null' + - string + - name: teams + type: DeclarativeStream + retriever: + type: SimpleRetriever + paginator: + type: NoPagination + requester: + path: teams + type: HttpRequester + url_base: https://api.assembledhq.com/v0/ + http_method: GET + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + request_headers: {} + request_body_json: {} + request_parameters: + limit: '500' + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - teams + - '*' + partition_router: [] + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + id: + type: string + name: + type: string + parent_id: + type: + - 'null' + - string + created_at: + type: number + updated_at: + type: number + - name: sites + type: DeclarativeStream + retriever: + type: SimpleRetriever + paginator: + type: NoPagination + requester: + path: sites + type: HttpRequester + url_base: https://api.assembledhq.com/v0/ + http_method: GET + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + request_headers: {} + request_body_json: {} + request_parameters: + limit: '500' + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - sites + - '*' + partition_router: [] + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + id: + type: string + name: + type: string + created_at: + type: number + updated_at: + type: number + - name: queues + type: DeclarativeStream + retriever: + type: SimpleRetriever + paginator: + type: NoPagination + requester: + path: queues + type: HttpRequester + url_base: https://api.assembledhq.com/v0/ + http_method: GET + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + request_headers: {} + request_body_json: {} + request_parameters: + limit: '500' + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - queues + - '*' + partition_router: [] + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + id: + type: string + name: + type: string + parent_id: + type: + - 'null' + - string + created_at: + type: number + updated_at: + type: number + - name: activity_types + type: DeclarativeStream + retriever: + type: SimpleRetriever + paginator: + type: NoPagination + requester: + path: activity_types + type: HttpRequester + url_base: https://api.assembledhq.com/v0/ + http_method: GET + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + request_headers: {} + request_body_json: {} + request_parameters: + limit: '500' + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - activity_types + - '*' + partition_router: [] + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + id: + type: string + name: + type: string + value: + type: string + timeoff: + type: boolean + channels: + type: array + items: + type: string + font_color: + type: + - 'null' + - string + productive: + type: boolean + short_name: + type: string + background_color: + type: string + - name: agent_states + type: DeclarativeStream + retriever: + type: SimpleRetriever + paginator: + type: DefaultPaginator + page_size_option: + type: RequestOption + field_name: limit + inject_into: request_parameter + page_token_option: + type: RequestOption + field_name: offset + inject_into: request_parameter + pagination_strategy: + type: OffsetIncrement + page_size: 500 + requester: + path: agents/state + type: HttpRequester + url_base: https://api.assembledhq.com/v0/ + http_method: GET + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + request_headers: {} + request_body_json: {} + request_parameters: {} + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - agent_states + - '*' + partition_router: [] + primary_key: + - agent_id + - start_time + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + state: + type: string + agent_id: + type: string + end_time: + type: number + platform: + type: string + ticket_id: + type: string + agent_name: + type: string + start_time: + type: number + agent_email: + type: string + external_id: + type: string + modified_at: + type: number + ticket_status: + type: string + agent_platform_id: + type: string + incremental_sync: + type: DatetimeBasedCursor + cursor_field: modified_at + end_datetime: + type: MinMaxDatetime + datetime: '{{ now_utc().strftime(''%Y-%m-%dT%H:%M:%SZ'') }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' + start_datetime: + type: MinMaxDatetime + datetime: '{{ config[''start_date''] }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: '%s' + end_time_option: + type: RequestOption + field_name: end_time + inject_into: request_parameter + lookback_window: P1M + start_time_option: + type: RequestOption + field_name: start_time + inject_into: request_parameter + cursor_datetime_formats: + - '%s' + - name: activities + type: DeclarativeStream + retriever: + type: SimpleRetriever + paginator: + type: NoPagination + requester: + path: activities + type: HttpRequester + url_base: https://api.assembledhq.com/v0/ + http_method: GET + authenticator: + type: BasicHttpAuthenticator + password: '{{ config[''password''] }}' + username: '{{ config[''username''] }}' + request_headers: {} + request_body_json: {} + request_parameters: + end_time: '2051222400' + include_agents: 'false' + include_activity_types: 'false' + record_selector: + type: RecordSelector + extractor: + type: DpathExtractor + field_path: + - activities + - '*' + partition_router: [] + primary_key: + - id + schema_loader: + type: InlineSchemaLoader + schema: + type: object + $schema: http://json-schema.org/schema# + properties: + id: + type: string + type_id: + type: string + agent_id: + type: string + end_time: + type: number + created_at: + type: number + start_time: + type: number + updated_at: + type: number + description: + type: + - 'null' + - string + incremental_sync: + type: DatetimeBasedCursor + cursor_field: updated_at + start_datetime: + type: MinMaxDatetime + datetime: '{{ config[''start_date''] }}' + datetime_format: '%Y-%m-%dT%H:%M:%SZ' + datetime_format: '%s' + lookback_window: P3D + start_time_option: + type: RequestOption + field_name: start_time + inject_into: request_parameter + cursor_datetime_formats: + - '%s' +version: 0.57.0 +metadata: + autoImportSchema: + sites: true + teams: true + people: true + queues: true + activities: true + agent_states: true + activity_types: true diff --git a/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/run.py b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/run.py new file mode 100644 index 000000000000..9a4b292bc823 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/run.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from .source import SourceAssembledCustom + +def run(): + source = SourceAssembledCustom() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/TODO.md b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/TODO.md new file mode 100644 index 000000000000..33a67a1bf540 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/TODO.md @@ -0,0 +1,16 @@ +# TODO: Define your stream schemas +Your connector must describe the schema of each stream it can output using [JSONSchema](https://json-schema.org). + +You can describe the schema of your streams using one `.json` file per stream. + +## Static schemas +From the `assembled_custom.yaml` configuration file, you read the `.json` files in the `schemas/` directory. You can refer to a schema in your configuration file using the `schema_loader` component's `file_path` field. For example: +``` +schema_loader: + type: JsonSchema + file_path: "./source_assembled_custom/schemas/customers.json" +``` +Every stream specified in the configuration file should have a corresponding `.json` schema file. + +Delete this file once you're done. Or don't. Up to you :) + diff --git a/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/customers.json b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/customers.json new file mode 100644 index 000000000000..9a4b13485836 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/customers.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "signup_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/employees.json b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/employees.json new file mode 100644 index 000000000000..2fa01a0fa1ff --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/schemas/employees.json @@ -0,0 +1,19 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "id": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "years_of_service": { + "type": ["null", "integer"] + }, + "start_date": { + "type": ["null", "string"], + "format": "date-time" + } + } +} diff --git a/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/source.py b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/source.py new file mode 100644 index 000000000000..94bd81e8a9f6 --- /dev/null +++ b/airbyte-integrations/connectors/source-assembled-custom/source_assembled_custom/source.py @@ -0,0 +1,18 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + +from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource + +""" +This file provides the necessary constructs to interpret a provided declarative YAML configuration file into +source connector. + +WARNING: Do not modify this file. +""" + + +# Declarative Source +class SourceAssembledCustom(YamlDeclarativeSource): + def __init__(self): + super().__init__(**{"path_to_yaml": "manifest.yaml"})