Merge branch 'master' into feature/source-commcare

DalgoT4D · Aug 16, 2024 · 87ea73b · 87ea73b
2 parents f42494a + 7b4c089
commit 87ea73b
Show file tree

Hide file tree

Showing 75 changed files with 4,168 additions and 231 deletions.
diff --git a/airbyte-integrations/connectors/source-avni/.dockerignore b/airbyte-integrations/connectors/source-avni/.dockerignore
@@ -0,0 +1,6 @@
+*
+!Dockerfile
+!main.py
+!source_avni
+!setup.py
+!secrets
diff --git a/airbyte-integrations/connectors/source-avni/Dockerfile b/airbyte-integrations/connectors/source-avni/Dockerfile
@@ -0,0 +1,37 @@
+FROM python:3.9.11-alpine3.15 as base
+
+# Build and load all requirements
+FROM base as builder
+WORKDIR /airbyte/integration_code
+
+# Upgrade pip to the latest version and install build dependencies
+RUN apk --no-cache upgrade \
+    && pip install --upgrade pip \
+    && apk --no-cache add tzdata build-base libffi-dev openssl-dev
+
+COPY setup.py ./
+# Install necessary packages to a temporary folder
+RUN pip install --prefix=/install .
+
+# Build a clean environment
+FROM base
+WORKDIR /airbyte/integration_code
+
+# Copy all loaded and built libraries to a pure basic image
+COPY --from=builder /install /usr/local
+# Add default timezone settings
+COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime
+RUN echo "Etc/UTC" > /etc/timezone
+
+# Bash is installed for more convenient debugging.
+RUN apk --no-cache add bash
+
+# Copy payload code only
+COPY main.py ./
+COPY source_avni ./source_avni
+
+ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
+ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
+
+LABEL io.airbyte.version=0.1.1
+LABEL io.airbyte.name=airbyte/source-avni
diff --git a/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml b/airbyte-integrations/connectors/source-avni/acceptance-test-config.yml
@@ -19,7 +19,7 @@ acceptance_tests:
       - config_path: "secrets/config.json"
         configured_catalog_path: "integration_tests/configured_catalog.json"
         empty_streams: []
-  incremental:
+  incremental: 
     tests:
       - config_path: "secrets/config.json"
         configured_catalog_path: "integration_tests/configured_catalog.json"

diff --git a/airbyte-integrations/connectors/source-avni/build.gradle b/airbyte-integrations/connectors/source-avni/build.gradle
@@ -0,0 +1,9 @@
+plugins {
+    id 'airbyte-python'
+    id 'airbyte-docker'
+    id 'airbyte-connector-acceptance-test'
+}
+
+airbytePython {
+    moduleDirectory 'source_avni'
+}
diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-avni/integration_tests/abnormal_state.json
@@ -11,4 +11,4 @@
   "encounters": {
     "last_modified_at": "2200-06-27T04:18:36.914Z"
   }
-}
+}
diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-avni/integration_tests/configured_catalog.json
@@ -6,63 +6,123 @@
         "json_schema": {
           "properties": {}
         },
-        "supported_sync_modes": ["full_refresh", "incremental"],
+        "supported_sync_modes": [
+          "full_refresh",
+          "incremental"
+        ],
         "source_defined_cursor": true,
-        "default_cursor_field": ["last_modified_at"],
-        "source_defined_primary_key": [["ID"]]
+        "default_cursor_field": [
+          "last_modified_at"
+        ],
+        "source_defined_primary_key": [
+          [
+            "ID"
+          ]
+        ]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append_dedup",
-      "cursor_field": ["last_modified_at"],
-      "primary_key": [["ID"]]
+      "cursor_field": [
+        "last_modified_at"
+      ],
+      "primary_key": [
+        [
+          "ID"
+        ]
+      ]
     },
     {
       "stream": {
         "name": "program_enrolments",
         "json_schema": {
           "properties": {}
         },
-        "supported_sync_modes": ["full_refresh", "incremental"],
+        "supported_sync_modes": [
+          "full_refresh",
+          "incremental"
+        ],
         "source_defined_cursor": true,
-        "default_cursor_field": ["last_modified_at"],
-        "source_defined_primary_key": [["ID"]]
+        "default_cursor_field": [
+          "last_modified_at"
+        ],
+        "source_defined_primary_key": [
+          [
+            "ID"
+          ]
+        ]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append_dedup",
-      "cursor_field": ["last_modified_at"],
-      "primary_key": [["ID"]]
+      "cursor_field": [
+        "last_modified_at"
+      ],
+      "primary_key": [
+        [
+          "ID"
+        ]
+      ]
     },
     {
       "stream": {
         "name": "program_encounters",
         "json_schema": {
           "properties": {}
         },
-        "supported_sync_modes": ["full_refresh", "incremental"],
+        "supported_sync_modes": [
+          "full_refresh",
+          "incremental"
+        ],
         "source_defined_cursor": true,
-        "default_cursor_field": ["last_modified_at"],
-        "source_defined_primary_key": [["ID"]]
+        "default_cursor_field": [
+          "last_modified_at"
+        ],
+        "source_defined_primary_key": [
+          [
+            "ID"
+          ]
+        ]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append_dedup",
-      "cursor_field": ["last_modified_at"],
-      "primary_key": [["ID"]]
+      "cursor_field": [
+        "last_modified_at"
+      ],
+      "primary_key": [
+        [
+          "ID"
+        ]
+      ]
     },
     {
       "stream": {
         "name": "encounters",
         "json_schema": {
           "properties": {}
         },
-        "supported_sync_modes": ["full_refresh", "incremental"],
+        "supported_sync_modes": [
+          "full_refresh",
+          "incremental"
+        ],
         "source_defined_cursor": true,
-        "default_cursor_field": ["last_modified_at"],
-        "source_defined_primary_key": [["ID"]]
+        "default_cursor_field": [
+          "last_modified_at"
+        ],
+        "source_defined_primary_key": [
+          [
+            "ID"
+          ]
+        ]
       },
       "sync_mode": "incremental",
       "destination_sync_mode": "append_dedup",
-      "cursor_field": ["last_modified_at"],
-      "primary_key": [["ID"]]
+      "cursor_field": [
+        "last_modified_at"
+      ],
+      "primary_key": [
+        [
+          "ID"
+        ]
+      ]
     }
   ]
-}
+}
diff --git a/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json b/airbyte-integrations/connectors/source-avni/integration_tests/sample_state.json
@@ -11,4 +11,4 @@
   "encounters": {
     "last_modified_at": "2000-06-27T04:18:36.914Z"
   }
-}
+}
diff --git a/airbyte-integrations/connectors/source-avni/metadata.yaml b/airbyte-integrations/connectors/source-avni/metadata.yaml
@@ -16,7 +16,7 @@ data:
     baseImage: docker.io/airbyte/python-connector-base:1.2.0@sha256:c22a9d97464b69d6ef01898edf3f8612dc11614f05a84984451dde195f337db9
   connectorType: source
   definitionId: 5d297ac7-355e-4a04-be75-a5e7e175fc4e
-  dockerImageTag: 0.1.0
+  dockerImageTag: 0.1.1
   dockerRepository: airbyte/source-avni
   githubIssueLabel: source-avni
   icon: avni.svg

diff --git a/airbyte-integrations/connectors/source-avni/setup.py b/airbyte-integrations/connectors/source-avni/setup.py
@@ -0,0 +1,32 @@
+#
+# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
+#
+
+
+from setuptools import find_packages, setup
+
+MAIN_REQUIREMENTS = [
+    "airbyte-cdk~=0.1",
+    "boto3==1.18.0",
+    "pytz==2024.1",
+]
+
+TEST_REQUIREMENTS = [
+    "requests-mock~=1.9.3",
+    "pytest~=6.2",
+    "pytest-mock~=3.6.1",
+    "connector-acceptance-test",
+]
+
+setup(
+    name="source_avni",
+    description="Source implementation for Avni.",
+    author="Airbyte",
+    author_email="[email protected]",
+    packages=find_packages(),
+    install_requires=MAIN_REQUIREMENTS,
+    package_data={"": ["*.json", "*.yaml", "schemas/*.json", "schemas/shared/*.json"]},
+    extras_require={
+        "tests": TEST_REQUIREMENTS,
+    },
+)
diff --git a/airbyte-integrations/connectors/source-avni/source_avni/components.py b/airbyte-integrations/connectors/source-avni/source_avni/components.py
@@ -1,14 +1,12 @@
-# Copyright (c) 2024 Airbyte, Inc., all rights reserved.
-
+from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator
 from dataclasses import dataclass
-
 import boto3
 import requests
-from airbyte_cdk.sources.declarative.auth.token import BasicHttpAuthenticator
 
 
 @dataclass
 class CustomAuthenticator(BasicHttpAuthenticator):
+
     @property
     def token(self) -> str:
 
@@ -30,8 +28,8 @@ def auth_header(self) -> str:
 
     def get_client_id(self):
 
-        url_client = "https://app.avniproject.org/idp-details"
-        response = requests.get(url_client)
+        url_client = self.config["url_base"] + "/idp-details"
+        response = requests.get(url_client, timeout=30)
         response.raise_for_status()
         client = response.json()
         return client["cognito"]["clientId"]
-Original file line number
+Diff line change
@@ Expand Up / @@ -11,4 +11,4 @@ @@
       "encounters": {
         "last_modified_at": "2200-06-27T04:18:36.914Z"
       }
-    }
+    }