-
Notifications
You must be signed in to change notification settings - Fork 1.4k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
13 changed files
with
404 additions
and
3 deletions.
There are no files selected for viewing
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
30 changes: 30 additions & 0 deletions
30
posthog/migrations/0466_alter_externaldatasource_source_type.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| # Generated by Django 4.2.15 on 2024-09-05 10:44 | ||
|
|
||
| from django.db import migrations, models | ||
|
|
||
|
|
||
| class Migration(migrations.Migration): | ||
| dependencies = [ | ||
| ("posthog", "0465_datawarehouse_stripe_account"), | ||
| ] | ||
|
|
||
| operations = [ | ||
| migrations.AlterField( | ||
| model_name="externaldatasource", | ||
| name="source_type", | ||
| field=models.CharField( | ||
| choices=[ | ||
| ("Stripe", "Stripe"), | ||
| ("Hubspot", "Hubspot"), | ||
| ("Postgres", "Postgres"), | ||
| ("Zendesk", "Zendesk"), | ||
| ("Snowflake", "Snowflake"), | ||
| ("Salesforce", "Salesforce"), | ||
| ("MySQL", "MySQL"), | ||
| ("MSSQL", "MSSQL"), | ||
| ("Vitally", "Vitally"), | ||
| ], | ||
| max_length=128, | ||
| ), | ||
| ), | ||
| ] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
246 changes: 246 additions & 0 deletions
246
posthog/temporal/data_imports/pipelines/vitally/__init__.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,246 @@ | ||
| import base64 | ||
| from typing import Any, Optional | ||
| import dlt | ||
| from dlt.sources.helpers.rest_client.paginators import BasePaginator | ||
| from dlt.sources.helpers.requests import Response, Request | ||
| import requests | ||
| from posthog.temporal.data_imports.pipelines.rest_source import RESTAPIConfig, rest_api_resources | ||
| from posthog.temporal.data_imports.pipelines.rest_source.typing import EndpointResource | ||
|
|
||
|
|
||
| def get_resource(name: str, is_incremental: bool) -> EndpointResource: | ||
| resources: dict[str, EndpointResource] = { | ||
| "Organizations": { | ||
| "name": "Organizations", | ||
| "table_name": "organizations", | ||
| "primary_key": "id", | ||
| "write_disposition": { | ||
| "disposition": "merge", | ||
| "strategy": "upsert", | ||
| } | ||
| if is_incremental | ||
| else "replace", | ||
| "endpoint": { | ||
| "data_selector": "results", | ||
| "path": "/resources/organizations", | ||
| "params": {"limit": 100, "sortBy": "updatedAt"}, | ||
| }, | ||
| "table_format": "delta", | ||
| }, | ||
| "Accounts": { | ||
| "name": "Accounts", | ||
| "table_name": "accounts", | ||
| "primary_key": "id", | ||
| "write_disposition": { | ||
| "disposition": "merge", | ||
| "strategy": "upsert", | ||
| } | ||
| if is_incremental | ||
| else "replace", | ||
| "endpoint": { | ||
| "data_selector": "results", | ||
| "path": "/resources/accounts", | ||
| "params": {"limit": 100, "sortBy": "updatedAt"}, | ||
| }, | ||
| "table_format": "delta", | ||
| }, | ||
| "Users": { | ||
| "name": "Users", | ||
| "table_name": "users", | ||
| "primary_key": "id", | ||
| "write_disposition": { | ||
| "disposition": "merge", | ||
| "strategy": "upsert", | ||
| } | ||
| if is_incremental | ||
| else "replace", | ||
| "endpoint": { | ||
| "data_selector": "results", | ||
| "path": "/resources/users", | ||
| "params": {"limit": 100, "sortBy": "updatedAt"}, | ||
| }, | ||
| "table_format": "delta", | ||
| }, | ||
| "Conversations": { | ||
| "name": "Conversations", | ||
| "table_name": "conversations", | ||
| "primary_key": "id", | ||
| "write_disposition": { | ||
| "disposition": "merge", | ||
| "strategy": "upsert", | ||
| } | ||
| if is_incremental | ||
| else "replace", | ||
| "endpoint": { | ||
| "data_selector": "results", | ||
| "path": "/resources/conversations", | ||
| "params": {"limit": 100, "sortBy": "updatedAt"}, | ||
| }, | ||
| "table_format": "delta", | ||
| }, | ||
| "Notes": { | ||
| "name": "Notes", | ||
| "table_name": "notes", | ||
| "primary_key": "id", | ||
| "write_disposition": { | ||
| "disposition": "merge", | ||
| "strategy": "upsert", | ||
| } | ||
| if is_incremental | ||
| else "replace", | ||
| "endpoint": { | ||
| "data_selector": "results", | ||
| "path": "/resources/notes", | ||
| "params": {"limit": 100, "sortBy": "updatedAt"}, | ||
| }, | ||
| "table_format": "delta", | ||
| }, | ||
| "Projects": { | ||
| "name": "Projects", | ||
| "table_name": "projects", | ||
| "primary_key": "id", | ||
| "write_disposition": { | ||
| "disposition": "merge", | ||
| "strategy": "upsert", | ||
| } | ||
| if is_incremental | ||
| else "replace", | ||
| "endpoint": { | ||
| "data_selector": "results", | ||
| "path": "/resources/projects", | ||
| "params": {"limit": 100, "sortBy": "updatedAt"}, | ||
| }, | ||
| "table_format": "delta", | ||
| }, | ||
| "Tasks": { | ||
| "name": "Tasks", | ||
| "table_name": "tasks", | ||
| "primary_key": "id", | ||
| "write_disposition": { | ||
| "disposition": "merge", | ||
| "strategy": "upsert", | ||
| } | ||
| if is_incremental | ||
| else "replace", | ||
| "endpoint": { | ||
| "data_selector": "results", | ||
| "path": "/resources/tasks", | ||
| "params": {"limit": 100, "sortBy": "updatedAt"}, | ||
| }, | ||
| "table_format": "delta", | ||
| }, | ||
| "NPS_Responses": { | ||
| "name": "NPS_Responses", | ||
| "table_name": "nps_responses", | ||
| "primary_key": "id", | ||
| "write_disposition": { | ||
| "disposition": "merge", | ||
| "strategy": "upsert", | ||
| } | ||
| if is_incremental | ||
| else "replace", | ||
| "endpoint": { | ||
| "data_selector": "results", | ||
| "path": "/resources/npsResponses", | ||
| "params": {"limit": 100, "sortBy": "updatedAt"}, | ||
| }, | ||
| "table_format": "delta", | ||
| }, | ||
| "Custom_Objects": { | ||
| "name": "Custom_Objects", | ||
| "table_name": "custom_objects", | ||
| "primary_key": "id", | ||
| "write_disposition": { | ||
| "disposition": "merge", | ||
| "strategy": "upsert", | ||
| } | ||
| if is_incremental | ||
| else "replace", | ||
| "endpoint": { | ||
| "data_selector": "results", | ||
| "path": "/resources/customObjects", | ||
| "params": {"limit": 100, "sortBy": "updatedAt"}, | ||
| }, | ||
| "table_format": "delta", | ||
| }, | ||
| } | ||
|
|
||
| return resources[name] | ||
|
|
||
|
|
||
| class VitallyPaginator(BasePaginator): | ||
| def __init__(self) -> None: | ||
| super().__init__() | ||
|
|
||
| def update_state(self, response: Response, data: Optional[list[Any]] = None) -> None: | ||
| res = response.json() | ||
|
|
||
| self._cursor = None | ||
|
|
||
| if not res: | ||
| self._has_next_page = False | ||
| return | ||
|
|
||
| if res["next"]: | ||
| self._has_next_page = True | ||
| self._cursor = res["next"] | ||
| else: | ||
| self._has_next_page = False | ||
|
|
||
| def update_request(self, request: Request) -> None: | ||
| if request.params is None: | ||
| request.params = {} | ||
|
|
||
| request.params["from"] = self._cursor | ||
|
|
||
|
|
||
| def get_base_url(region: str, subdomain: Optional[str]) -> str: | ||
| if region == "US" and subdomain: | ||
| return f"https://{subdomain}.rest.vitally.io/" | ||
|
|
||
| return "https://rest.vitally-eu.io/" | ||
|
|
||
|
|
||
| @dlt.source(max_table_nesting=0) | ||
| def vitally_source( | ||
| secret_token: str, | ||
| region: str, | ||
| subdomain: Optional[str], | ||
| endpoint: str, | ||
| team_id: int, | ||
| job_id: str, | ||
| is_incremental: bool = False, | ||
| ): | ||
| config: RESTAPIConfig = { | ||
| "client": { | ||
| "base_url": get_base_url(region, subdomain), | ||
| "auth": { | ||
| "type": "http_basic", | ||
| "username": secret_token, | ||
| "password": "", | ||
| }, | ||
| "paginator": VitallyPaginator(), | ||
| }, | ||
| "resource_defaults": { | ||
| "primary_key": "id", | ||
| "write_disposition": { | ||
| "disposition": "merge", | ||
| "strategy": "upsert", | ||
| } | ||
| if is_incremental | ||
| else "replace", | ||
| }, | ||
| "resources": [get_resource(endpoint, is_incremental)], | ||
| } | ||
|
|
||
| yield from rest_api_resources(config, team_id, job_id) | ||
|
|
||
|
|
||
| def validate_credentials(secret_token: str, region: str, subdomain: Optional[str]) -> bool: | ||
| basic_token = base64.b64encode(f"{secret_token}:".encode("ascii")).decode("ascii") | ||
| res = requests.get( | ||
| f"{get_base_url(region, subdomain)}resources/users?limit=1", | ||
| headers={"Authorization": f"Basic {basic_token}"}, | ||
| ) | ||
|
|
||
| return res.status_code == 200 |
Oops, something went wrong.