From 9909cd69f240d9ca047c990cf546135ac1dbaf41 Mon Sep 17 00:00:00 2001 From: Serhii Chvaliuk Date: Thu, 13 Jan 2022 16:49:32 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20Source=20HubSpot:=20`engagements?= =?UTF-8?q?`=20stream,=20remove=20auto-generated=20`properties`=20(#9385)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * extend schemas/engagements.json * fix unix newlines Signed-off-by: Sergey Chvalyuk --- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 2 +- .../connectors/source-hubspot/Dockerfile | 2 +- .../source-hubspot/source_hubspot/api.py | 8 +- .../source_hubspot/schemas/engagements.json | 164 ++++++++++++++++++ docs/integrations/sources/hubspot.md | 14 +- 6 files changed, 176 insertions(+), 16 deletions(-) diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index d7ea734d5397..74993fe19c8c 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -287,7 +287,7 @@ - name: HubSpot sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c dockerRepository: airbyte/source-hubspot - dockerImageTag: 0.1.30 + dockerImageTag: 0.1.31 documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot icon: hubspot.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 7024482bc301..dd09676679a8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2846,7 +2846,7 @@ path_in_connector_config: - "credentials" - "client_secret" -- dockerImage: "airbyte/source-hubspot:0.1.30" +- dockerImage: "airbyte/source-hubspot:0.1.31" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot" connectionSpecification: diff --git a/airbyte-integrations/connectors/source-hubspot/Dockerfile b/airbyte-integrations/connectors/source-hubspot/Dockerfile index c8ddb3ac162e..b8bd788b20cd 100644 --- a/airbyte-integrations/connectors/source-hubspot/Dockerfile +++ b/airbyte-integrations/connectors/source-hubspot/Dockerfile @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.30 +LABEL io.airbyte.version=0.1.31 LABEL io.airbyte.name=airbyte/source-hubspot diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py index 7d02a46eacb5..708836d4d3b8 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py @@ -265,10 +265,7 @@ def _cast_value(declared_field_types: List, field_name: str, field_value: Any, d def _cast_record_fields_if_needed(self, record: Mapping, properties: Mapping[str, Any] = None) -> Mapping: - if self.entity not in {"contact", "engagement", "product", "quote", "ticket", "company", "deal", "line_item"}: - return record - - if not record.get("properties"): + if not self.entity or not record.get("properties"): return record properties = properties or self.properties @@ -363,7 +360,7 @@ def parse_response(self, response: Union[Mapping[str, Any], List[dict]]) -> Iter 'message': 'This hapikey (....) does not have proper permissions! (requires any of [automation-access])', 'correlationId': '111111-2222-3333-4444-55555555555'} """ - logger.warning(f"Stream `{self.entity}` cannot be procced. {response.get('message')}") + logger.warning(f"Stream `{self.name}` cannot be procced. {response.get('message')}") return if response.get(self.data_field) is None: @@ -713,7 +710,6 @@ class EngagementStream(Stream): Docs: https://legacydocs.hubspot.com/docs/methods/engagements/get-all-engagements """ - entity = "engagement" url = "/engagements/v1/engagements/paged" more_key = "hasMore" limit = 250 diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json index 364649a21081..d00ba5d6a342 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json @@ -5,6 +5,12 @@ "id": { "type": ["null", "integer"] }, + "uid": { + "type": ["null", "string"] + }, + "teamId": { + "type": ["null", "integer"] + }, "portalId": { "type": ["null", "integer"] }, @@ -14,6 +20,12 @@ "createdAt": { "type": ["null", "integer"] }, + "createdBy": { + "type": ["null", "integer"] + }, + "modifiedBy": { + "type": ["null", "integer"] + }, "lastUpdated": { "type": ["null", "integer"] }, @@ -26,6 +38,33 @@ "timestamp": { "type": ["null", "integer"] }, + "bodyPreview": { + "type": ["null", "string"] + }, + "bodyPreviewHtml": { + "type": ["null", "string"] + }, + "bodyPreviewIsTruncated": { + "type": ["null", "boolean"] + }, + "allAccessibleTeamIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "activityType": { + "type": ["null", "string"] + }, + "gdprDeleted": { + "type": ["null", "boolean"] + }, + "source": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + }, "associations": { "type": ["null", "object"], "properties": { @@ -46,6 +85,24 @@ "items": { "type": ["null", "integer"] } + }, + "ownerIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "workflowIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "ticketIds": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } } } }, @@ -77,6 +134,17 @@ }, "lastName": { "type": ["null", "string"] + }, + "raw": { + "type": ["null", "string"] + } + } + }, + "sender": { + "type": ["null", "object"], + "properties": { + "email": { + "type": ["null", "string"] } } }, @@ -87,6 +155,15 @@ "properties": { "email": { "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "raw": { + "type": ["null", "string"] } } } @@ -98,6 +175,15 @@ "properties": { "email": { "type": ["null", "string"] + }, + "firstName": { + "type": ["null", "string"] + }, + "lastName": { + "type": ["null", "string"] + }, + "raw": { + "type": ["null", "string"] } } } @@ -157,6 +243,84 @@ }, "disposition": { "type": ["null", "string"] + }, + "completionDate": { + "type": ["null", "integer"] + }, + "taskType": { + "type": ["null", "string"] + }, + "reminders": { + "type": ["null", "array"], + "items": { + "type": ["null", "integer"] + } + }, + "threadId": { + "type": ["null", "string", "integer"] + }, + "messageId": { + "type": ["null", "string"] + }, + "loggedFrom": { + "type": ["null", "string"] + }, + "attachedVideoOpened": { + "type": ["null", "boolean"] + }, + "attachedVideoWatched": { + "type": ["null", "boolean"] + }, + "trackerKey": { + "type": ["null", "string"] + }, + "sendDefaultReminder": { + "type": ["null", "boolean"] + }, + "source": { + "type": ["null", "string"] + }, + "unknownVisitorConversation": { + "type": ["null", "boolean"] + }, + "facsimileSendId": { + "type": ["null", "string"] + }, + "sentVia": { + "type": ["null", "string"] + }, + "sequenceStepOrder": { + "type": ["null", "integer"] + }, + "externalUrl": { + "type": ["null", "string"] + }, + "postSendStatus": { + "type": ["null", "string"] + }, + "errorMessage": { + "type": ["null", "string"] + }, + "recipientDropReasons": { + "type": ["null", "string"] + }, + "calleeObjectId": { + "type": ["null", "integer"] + }, + "calleeObjectType": { + "type": ["null", "string"] + }, + "mediaProcessingStatus": { + "type": ["null", "string"] + }, + "sourceId": { + "type": ["null", "string"] + }, + "priority": { + "type": ["null", "string"] + }, + "isAllDay": { + "type": ["null", "boolean"] } } } diff --git a/docs/integrations/sources/hubspot.md b/docs/integrations/sources/hubspot.md index 81c77c50f345..90ac117be7bf 100644 --- a/docs/integrations/sources/hubspot.md +++ b/docs/integrations/sources/hubspot.md @@ -37,15 +37,15 @@ This source is capable of syncing the following tables and their data: * [Workflows](https://legacydocs.hubspot.com/docs/methods/workflows/v3/get_workflows) ### A note on the `engagements` stream -Objects in the `engagements` stream can have one of the following types: `note`, `email`, `task`, `meeting`, `call`. +Objects in the `engagements` stream can have one of the following types: `note`, `email`, `task`, `meeting`, `call`. -Depending on the type of engagement, different properties will be set for that object in the `engagements_metadata` table in the destination. +Depending on the type of engagement, different properties will be set for that object in the `engagements_metadata` table in the destination. * A `call` engagement will have a corresponding `engagements_metadata` object with non-null values in the `toNumber`, `fromNumber`, `status`, `externalId`, `durationMilliseconds`, `externalAccountId`, `recordingUrl`, `body`, and `disposition` columns. -* An `email` engagement will have a corresponding `engagements_metadata` object with with non-null values in the `subject`, `html`, and `text` columns. In addition, there will be records in four related tables, `engagements_metadata_from`, `engagements_metadata_to`, `engagements_metadata_cc`, `engagements_metadata_bcc`. -* A `meeting` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `startTime`, `endTime`, and `title` columns. -* A `note` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body` column. -* A `task` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `status`, and `forObjectType` columns. +* An `email` engagement will have a corresponding `engagements_metadata` object with with non-null values in the `subject`, `html`, and `text` columns. In addition, there will be records in four related tables, `engagements_metadata_from`, `engagements_metadata_to`, `engagements_metadata_cc`, `engagements_metadata_bcc`. +* A `meeting` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `startTime`, `endTime`, and `title` columns. +* A `note` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body` column. +* A `task` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `status`, and `forObjectType` columns. **Note**: HubSpot API currently only supports `quotes` endpoint using API Key, using Oauth it is impossible to access this stream (as reported by [community.hubspot.com](https://community.hubspot.com/t5/APIs-Integrations/Help-with-using-Feedback-CRM-API-and-Quotes-CRM-API/m-p/449104/highlight/true#M44411)). @@ -110,6 +110,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | Version | Date | Pull Request | Subject | |:--------|:-----------| :--- |:-----------------------------------------------------------------------------------------------------------------------------------------------| +| 0.1.31 | 2022-01-11 | [9385](https://github.com/airbytehq/airbyte/pull/9385) | Remove auto-generated `properties` from `Engagements` stream | | 0.1.30 | 2021-01-10 | [9129](https://github.com/airbytehq/airbyte/pull/9129) | Created Contacts list memberships streams | | 0.1.29 | 2021-12-17 | [8699](https://github.com/airbytehq/airbyte/pull/8699) | Add incremental sync support for `companies`, `contact_lists`, `contacts`, `deals`, `line_items`, `products`, `quotes`, `tickets` streams | | 0.1.28 | 2021-12-15 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update fields and descriptions | @@ -134,4 +135,3 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt | 0.1.9 | 2021-08-11 | [5334](https://github.com/airbytehq/airbyte/pull/5334) | Fix empty strings inside float datatype | | 0.1.8 | 2021-08-06 | [5250](https://github.com/airbytehq/airbyte/pull/5250) | Fix issue with printing exceptions | | 0.1.7 | 2021-07-27 | [4913](https://github.com/airbytehq/airbyte/pull/4913) | Update fields schema | -