From 4a3702529aadde96890ae4eae3a5d32789aee7be Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Wed, 18 Sep 2024 19:51:03 +0530 Subject: [PATCH 1/2] current-avni connector --- .../connectors/source-avni/.dockerignore | 6 + .../connectors/source-avni/Dockerfile | 37 +++++ .../source-avni/acceptance-test-config.yml | 2 +- .../source-avni/acceptance-test-docker.sh | 0 .../connectors/source-avni/build.gradle | 9 ++ .../integration_tests/abnormal_state.json | 2 +- .../integration_tests/configured_catalog.json | 102 ++++++++++--- .../integration_tests/sample_state.json | 2 +- .../connectors/source-avni/metadata.yaml | 2 +- .../connectors/source-avni/setup.py | 32 ++++ .../source-avni/source_avni/components.py | 10 +- .../source-avni/source_avni/manifest.yaml | 49 ++++++- .../source_avni/schemas/approvalStatuses.json | 50 +++++++ .../source_avni/schemas/encounters.json | 102 ++++++++++--- .../source_avni/schemas/locations.json | 81 +++++++++++ .../schemas/program_encounters.json | 132 +++++++++++++---- .../schemas/program_enrolments.json | 102 ++++++++++--- .../source_avni/schemas/subjects.json | 137 ++++++++++++++---- .../source-avni/unit_tests/test_components.py | 25 ++-- 19 files changed, 733 insertions(+), 149 deletions(-) create mode 100644 airbyte-integrations/connectors/source-avni/.dockerignore create mode 100644 airbyte-integrations/connectors/source-avni/Dockerfile mode change 100755 => 100644 airbyte-integrations/connectors/source-avni/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-avni/build.gradle create mode 100644 airbyte-integrations/connectors/source-avni/setup.py create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json create mode 100644 airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json diff --git a/airbyte-integrations/connectors/source-avni/.dockerignore b/airbyte-integrations/connectors/source-avni/.dockerignore new file mode 100644 index 000000000000..3fcbe3fc3f0b --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/.dockerignore @@ -0,0 +1,6 @@ +* +!Dockerfile +!main.py +!source_avni +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-avni/Dockerfile b/airbyte-integrations/connectors/source-avni/Dockerfile new file mode 100644 index 000000000000..7cabf21c80e5 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/Dockerfile @@ -0,0 +1,37 @@ +FROM python:3.9.11-alpine3.15 as base + +# Build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# Upgrade pip to the latest version and install build dependencies +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base libffi-dev openssl-dev + +COPY setup.py ./ +# Install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# Build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# Copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# Add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# Bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# Copy payload code only +COPY main.py ./ +COPY source_avni ./source_avni + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.1 +LABEL io.airbyte.name=airbyte/source-avni diff --git a/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml b/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml index ffd159d73dc1..54ec3ab7e71f 100644 --- a/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml +++ b/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml @@ -19,7 +19,7 @@ acceptance_tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" empty_streams: [] - incremental: + incremental: tests: - config_path: "secrets/config.json" configured_catalog_path: "integration_tests/configured_catalog.json" diff --git a/airbyte-integrations/connectors/source-avni/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-avni/acceptance-test-docker.sh old mode 100755 new mode 100644 diff --git a/airbyte-integrations/connectors/source-avni/build.gradle b/airbyte-integrations/connectors/source-avni/build.gradle new file mode 100644 index 000000000000..8b0332499806 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/build.gradle @@ -0,0 +1,9 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-connector-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_avni' +} diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json index 195c2660ca8b..7ee843e317cd 100644 --- a/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json @@ -11,4 +11,4 @@ "encounters": { "last_modified_at": "2200-06-27T04:18:36.914Z" } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json index 1e435deb4481..b9f2427c16b4 100644 --- a/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json @@ -6,15 +6,30 @@ "json_schema": { "properties": {} }, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], "source_defined_cursor": true, - "default_cursor_field": ["last_modified_at"], - "source_defined_primary_key": [["ID"]] + "default_cursor_field": [ + "last_modified_at" + ], + "source_defined_primary_key": [ + [ + "ID" + ] + ] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup", - "cursor_field": ["last_modified_at"], - "primary_key": [["ID"]] + "cursor_field": [ + "last_modified_at" + ], + "primary_key": [ + [ + "ID" + ] + ] }, { "stream": { @@ -22,15 +37,30 @@ "json_schema": { "properties": {} }, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], "source_defined_cursor": true, - "default_cursor_field": ["last_modified_at"], - "source_defined_primary_key": [["ID"]] + "default_cursor_field": [ + "last_modified_at" + ], + "source_defined_primary_key": [ + [ + "ID" + ] + ] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup", - "cursor_field": ["last_modified_at"], - "primary_key": [["ID"]] + "cursor_field": [ + "last_modified_at" + ], + "primary_key": [ + [ + "ID" + ] + ] }, { "stream": { @@ -38,15 +68,30 @@ "json_schema": { "properties": {} }, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], "source_defined_cursor": true, - "default_cursor_field": ["last_modified_at"], - "source_defined_primary_key": [["ID"]] + "default_cursor_field": [ + "last_modified_at" + ], + "source_defined_primary_key": [ + [ + "ID" + ] + ] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup", - "cursor_field": ["last_modified_at"], - "primary_key": [["ID"]] + "cursor_field": [ + "last_modified_at" + ], + "primary_key": [ + [ + "ID" + ] + ] }, { "stream": { @@ -54,15 +99,30 @@ "json_schema": { "properties": {} }, - "supported_sync_modes": ["full_refresh", "incremental"], + "supported_sync_modes": [ + "full_refresh", + "incremental" + ], "source_defined_cursor": true, - "default_cursor_field": ["last_modified_at"], - "source_defined_primary_key": [["ID"]] + "default_cursor_field": [ + "last_modified_at" + ], + "source_defined_primary_key": [ + [ + "ID" + ] + ] }, "sync_mode": "incremental", "destination_sync_mode": "append_dedup", - "cursor_field": ["last_modified_at"], - "primary_key": [["ID"]] + "cursor_field": [ + "last_modified_at" + ], + "primary_key": [ + [ + "ID" + ] + ] } ] -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json index 812c9b530911..9a3faaca99e0 100644 --- a/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json +++ b/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json @@ -11,4 +11,4 @@ "encounters": { "last_modified_at": "2000-06-27T04:18:36.914Z" } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/metadata.yaml b/airbyte-integrations/connectors/source-avni/metadata.yaml index 7a0ccb35aff2..87cd2e9a1a45 100644 --- a/airbyte-integrations/connectors/source-avni/metadata.yaml +++ b/airbyte-integrations/connectors/source-avni/metadata.yaml @@ -16,7 +16,7 @@ data: baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9 connectorType: source definitionId: 5d297ac7-355e-4a04-be75-a5e7e175fc4e - dockerImageTag: 0.1.0 + dockerImageTag: 0.1.1 dockerRepository: airbyte/source-avni githubIssueLabel: source-avni icon: avni.svg diff --git a/airbyte-integrations/connectors/source-avni/setup.py b/airbyte-integrations/connectors/source-avni/setup.py new file mode 100644 index 000000000000..4aead2c068a7 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/setup.py @@ -0,0 +1,32 @@ +# +# Copyright (c) 2023 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", + "boto3==1.18.0", + "pytz==2024.1", +] + +TEST_REQUIREMENTS = [ + "requests-mock~=1.9.3", + "pytest~=6.2", + "pytest-mock~=3.6.1", + "connector-acceptance-test", +] + +setup( + name="source_avni", + description="Source implementation for Avni.", + author="Airbyte", + author_email="contact@airbyte.io", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/components.py b/airbyte-integrations/connectors/source-avni/source_avni/components.py index d47cbf7654f6..1f7f7f3cf873 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/components.py +++ b/airbyte-integrations/connectors/source-avni/source_avni/components.py @@ -1,14 +1,12 @@ -# Copyright (c) 2024 Airbyte, Inc., all rights reserved. - +from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator from dataclasses import dataclass - import boto3 import requests -from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator @dataclass class CustomAuthenticator(BasicHttpAuthenticator): + @property def token(self) -> str: @@ -30,8 +28,8 @@ def auth_header(self) -> str: def get_client_id(self): - url_client = "https://app.avniproject.org/idp-details" - response = requests.get(url_client) + url_client = self.config["url_base"] + "/idp-details" + response = requests.get(url_client, timeout=30) response.raise_for_status() client = response.json() return client["cognito"]["clientId"] diff --git a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml index 5f8566c5b0ad..765f0fee5b05 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml +++ b/airbyte-integrations/connectors/source-avni/source_avni/manifest.yaml @@ -9,12 +9,13 @@ definitions: requester: type: HttpRequester - url_base: "https://app.avniproject.org/api" + url_base: "{{config['url_base']}}/api" http_method: "GET" authenticator: class_name: source_avni.components.CustomAuthenticator username: "{{config['username']}}" password: "{{config['password']}}" + idp_base: "{{config['url_base']}}" retriever: type: SimpleRetriever @@ -28,7 +29,7 @@ definitions: field_name: "size" pagination_strategy: type: "PageIncrement" - page_size: 100 + page_size: 1000 page_token_option: type: "RequestOption" inject_into: "request_parameter" @@ -48,10 +49,10 @@ definitions: inject_into: "request_parameter" transformations_base: - - type: AddFields - fields: - - path: ["last_modified_at"] - value: "{{ record['audit']['Last modified at'] }}" + - type: AddFields + fields: + - path: [ "last_modified_at" ] + value: "{{ record['audit']['Last modified at'] }}" base_stream: type: DeclarativeStream @@ -102,11 +103,35 @@ definitions: $parameters: path: "/encounters" + approval_status_stream: + $ref: "#/definitions/base_stream" + name: "approvalStatuses" + primary_key: "Entity ID" + incremental_sync: + $ref: "#/definitions/incremental_base" + transformations: + $ref: "#/definitions/transformations_base" + $parameters: + path: "/approvalStatuses" + + locations_stream: + $ref: "#/definitions/base_stream" + name: "locations" + primary_key: "ID" + incremental_sync: + $ref: "#/definitions/incremental_base" + transformations: + $ref: "#/definitions/transformations_base" + $parameters: + path: "/locations" + streams: - "#/definitions/subjects_stream" - "#/definitions/program_enrolments_stream" - "#/definitions/program_encounters_stream" - "#/definitions/encounters_stream" + - "#/definitions/approval_status_stream" + - "#/definitions/locations_stream" check: type: CheckStream @@ -122,19 +147,31 @@ spec: required: - username - password + - url_base - start_date additionalProperties: true properties: username: type: string description: Your avni platform Username + title: Username + order: 1 password: type: string description: Your avni platform password + title: Password airbyte_secret: true + order: 2 + url_base: + type: string + description: Your avni platform base url, with no trailing slash (/) + title: Base URL (no trailing /) + order: 3 start_date: type: string default: "2000-06-23T01:30:00.000Z" description: Specify Date and time from which you want to fetch data + title: Start Date + order: 4 examples: - "2000-10-31T01:30:00.000Z" diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json new file mode 100644 index 000000000000..f5241fe241d4 --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/approvalStatuses.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "Entity ID": { + "type": "string" + }, + "Entity type": { + "type": "string" + }, + "Entity type ID": { + "type": "string" + }, + "Approval status": { + "type": "string" + }, + "Approval status comment": { + "type": ["null", "string"] + }, + "Status date time": { + "type": "string", + "format": "date-time" + }, + "last_modified_at": { + "type": "string", + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "audit": { + "type": "object", + "properties": { + "Created at": { + "type": "string", + "format": "date-time" + }, + "Last modified at": { + "type": ["null", "string"], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Created by": { + "type": "string" + }, + "Last modified by": { + "type": "string" + } + } + } + } + } + diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json index 4e76035f17c0..54b1ea1a01c6 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/encounters.json @@ -7,13 +7,19 @@ "type": "string" }, "External ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Voided": { "type": "boolean" }, "Encounter type": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Subject ID": { "type": "string" @@ -22,55 +28,94 @@ "type": "string" }, "Subject external ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Encounter date time": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Encounter location": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "X": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] }, "Y": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] } } }, "Earliest scheduled date": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Max scheduled date": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "observations": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true }, "Cancel location": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "X": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] }, "Y": { - "type": ["null", "number"], + "type": [ + "null", + "number" + ], "example": 74.7364501 } } }, "Cancel date time": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "cancelObservations": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true }, "last_modified_at": { @@ -78,24 +123,39 @@ "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "audit": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "Created at": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Last modified at": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Created by": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Last modified by": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json new file mode 100644 index 000000000000..354076ab700b --- /dev/null +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/locations.json @@ -0,0 +1,81 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "ID": { + "type": "string" + }, + "External ID": { + "type": [ + "null", + "string" + ] + }, + "Title": { + "type": "string" + }, + "Type": { + "type": "string" + }, + "Level": { + "type": "number" + }, + "Voided": { + "type": "boolean" + }, + "customProperties": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + }, + "last_modified_at": { + "type": "string", + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Parent": { + "type": [ + "null", + "object" + ], + "additionalProperties": true + }, + "audit": { + "type": [ + "null", + "object" + ], + "additionalProperties": true, + "properties": { + "Created at": { + "type": [ + "null", + "string" + ], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Last modified at": { + "type": [ + "null", + "string" + ], + "format": "YYYY-MM-DDTHH:mm:ss.sssZ" + }, + "Created by": { + "type": [ + "null", + "string" + ] + }, + "Last modified by": { + "type": [ + "null", + "string" + ] + } + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_encounters.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_encounters.json index ce32347b7495..f898b3d69474 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_encounters.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_encounters.json @@ -4,83 +4,146 @@ "additionalProperties": true, "properties": { "ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "External ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Voided": { "type": "boolean" }, "Subject ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Subject type": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Subject external ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Enrolment ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Enrolment external ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Program": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Encounter type": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Encounter date time": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Encounter location": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "X": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] }, "Y": { - "type": ["null", "number"], + "type": [ + "null", + "number" + ], "example": 74.7364501 } } }, "Earliest scheduled date": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Max scheduled date": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "observations": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true }, "Cancel location": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "X": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] }, "Y": { - "type": ["null", "number"], + "type": [ + "null", + "number" + ], "example": 74.7364501 } } }, "Cancel date time": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "cancelObservations": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true }, "last_modified_at": { @@ -88,23 +151,38 @@ "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "audit": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "properties": { "Created at": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Last modified at": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Created by": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Last modified by": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_enrolments.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_enrolments.json index bbfe5d262c55..b890031b7ec4 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_enrolments.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/program_enrolments.json @@ -7,7 +7,10 @@ "type": "string" }, "External ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Voided": { "type": "boolean" @@ -19,53 +22,95 @@ "type": "string" }, "Subject external ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Program": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Enrolment datetime": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Enrolment location": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "X": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] }, "Y": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] } } }, "Exit datetime": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Exit location": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "X": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] }, "Y": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] } } }, "observations": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true }, "exitObservations": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true }, "encounters": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } }, "last_modified_at": { @@ -73,24 +118,39 @@ "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "audit": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "Created at": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Last modified at": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Created by": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Last modified by": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json index 9b1d74bb7f0f..f8f4bcbb10a7 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json @@ -7,36 +7,60 @@ "type": "string" }, "External ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Voided": { "type": "boolean" }, "Subject type": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Registration location": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "X": { - "type": ["null", "number"] + "type": [ + "null", + "number" + ] }, "Y": { - "type": ["null", "number"], + "type": [ + "null", + "number" + ], "example": 74.7364501 } } }, "Registration date": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "location": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true }, "relatives": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { "type": "object", "additionalProperties": true, @@ -45,38 +69,68 @@ "type": "boolean" }, "Relationship type": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Relative ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Relative external ID": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Enter date": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Exit date": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } }, "observations": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true }, "encounters": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "uuid" } }, "enrolments": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "uuid" } }, @@ -85,36 +139,63 @@ "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "catchments": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } }, "audit": { - "type": ["null", "object"], + "type": [ + "null", + "object" + ], "additionalProperties": true, "properties": { "Created at": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Last modified at": { - "type": ["null", "string"], + "type": [ + "null", + "string" + ], "format": "YYYY-MM-DDTHH:mm:ss.sssZ" }, "Created by": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] }, "Last modified by": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } }, "Groups": { - "type": ["null", "array"], + "type": [ + "null", + "array" + ], "items": { - "type": ["null", "string"] + "type": [ + "null", + "string" + ] } } } -} +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py b/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py index 49f77bed58ec..983984d33726 100644 --- a/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py +++ b/airbyte-integrations/connectors/source-avni/unit_tests/test_components.py @@ -7,14 +7,14 @@ from source_avni.components import CustomAuthenticator -@patch('boto3.client') +@patch("boto3.client") def test_token_property(mock_boto3_client): mock_cognito_client = Mock() mock_boto3_client.return_value = mock_cognito_client - config= { "username": "example@gmail.com", "api_key": "api_key" } - source = CustomAuthenticator(config=config,username="example@gmail.com",password="api_key",parameters="") + config = {"username": "example@gmail.com", "api_key": "api_key"} + source = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters="") source._username = Mock() source._username.eval.return_value = "test_username" source._password = Mock() @@ -22,24 +22,19 @@ def test_token_property(mock_boto3_client): source.get_client_id = Mock() source.get_client_id.return_value = "test_client_id" - mock_cognito_client.initiate_auth.return_value = { - "AuthenticationResult": { - "IdToken": "test_id_token" - } - } + mock_cognito_client.initiate_auth.return_value = {"AuthenticationResult": {"IdToken": "test_id_token"}} token = source.token mock_boto3_client.assert_called_once_with("cognito-idp", region_name="ap-south-1") mock_cognito_client.initiate_auth.assert_called_once_with( - ClientId="test_client_id", - AuthFlow="USER_PASSWORD_AUTH", - AuthParameters={"USERNAME": "test_username", "PASSWORD": "test_password"} + ClientId="test_client_id", AuthFlow="USER_PASSWORD_AUTH", AuthParameters={"USERNAME": "test_username", "PASSWORD": "test_password"} ) assert token == "test_id_token" + def test_get_client_id(mocker): - - config= { "username": "example@gmail.com", "api_key": "api_key" } - source = CustomAuthenticator(config=config,username="example@gmail.com",password="api_key",parameters="") + + config = {"username": "example@gmail.com", "api_key": "api_key"} + source = CustomAuthenticator(config=config, username="example@gmail.com", password="api_key", parameters="") client_id = source.get_client_id() expected_length = 26 - assert len(client_id) == expected_length \ No newline at end of file + assert len(client_id) == expected_length From 1be785d20c58e62eefcd90ce741aeaee1d4849bc Mon Sep 17 00:00:00 2001 From: Ishankoradia Date: Wed, 18 Sep 2024 20:14:25 +0530 Subject: [PATCH 2/2] added location id in subjects schema --- .../source-avni/source_avni/schemas/subjects.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json b/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json index f8f4bcbb10a7..8f5cb18292a1 100644 --- a/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json +++ b/airbyte-integrations/connectors/source-avni/source_avni/schemas/subjects.json @@ -56,6 +56,12 @@ ], "additionalProperties": true }, + "Location ID": { + "type": [ + "null", + "string" + ] + }, "relatives": { "type": [ "null",