Skip to content

Commit

Permalink
🎉 Source HubSpot: engagements stream, remove auto-generated `proper…
Browse files Browse the repository at this point in the history 
…ties` (airbytehq#9385)

* extend schemas/engagements.json

* fix unix newlines

Signed-off-by: Sergey Chvalyuk <[email protected]>
  • Loading branch information
@grubberr
grubberr authored Jan 13, 2022
1 parent 4534703 commit 9909cd6
Show file tree
Hide file tree
Showing 6 changed files with 176 additions and 16 deletions.
2 changes: 1 addition & 1 deletion airbyte-config/init/src/main/resources/seed/source_definitions.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,7 +287,7 @@
- name: HubSpot
sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c
dockerRepository: airbyte/source-hubspot
dockerImageTag: 0.1.30
dockerImageTag: 0.1.31
documentationUrl: https://docs.airbyte.io/integrations/sources/hubspot
icon: hubspot.svg
sourceType: api
Expand Down
2 changes: 1 addition & 1 deletion airbyte-config/init/src/main/resources/seed/source_specs.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2846,7 +2846,7 @@
path_in_connector_config:
- "credentials"
- "client_secret"
- dockerImage: "airbyte/source-hubspot:0.1.30"
- dockerImage: "airbyte/source-hubspot:0.1.31"
spec:
documentationUrl: "https://docs.airbyte.io/integrations/sources/hubspot"
connectionSpecification:
Expand Down
2 changes: 1 addition & 1 deletion airbyte-integrations/connectors/source-hubspot/Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,5 +34,5 @@ COPY source_hubspot ./source_hubspot
ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]

LABEL io.airbyte.version=0.1.30
LABEL io.airbyte.version=0.1.31
LABEL io.airbyte.name=airbyte/source-hubspot
8 changes: 2 additions & 6 deletions airbyte-integrations/connectors/source-hubspot/source_hubspot/api.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -265,10 +265,7 @@ def _cast_value(declared_field_types: List, field_name: str, field_value: Any, d

def _cast_record_fields_if_needed(self, record: Mapping, properties: Mapping[str, Any] = None) -> Mapping:

if self.entity not in {"contact", "engagement", "product", "quote", "ticket", "company", "deal", "line_item"}:
return record

if not record.get("properties"):
if not self.entity or not record.get("properties"):
return record

properties = properties or self.properties
Expand Down Expand Up @@ -363,7 +360,7 @@ def parse_response(self, response: Union[Mapping[str, Any], List[dict]]) -> Iter
'message': 'This hapikey (....) does not have proper permissions! (requires any of [automation-access])',
'correlationId': '111111-2222-3333-4444-55555555555'}
"""
logger.warning(f"Stream `{self.entity}` cannot be procced. {response.get('message')}")
logger.warning(f"Stream `{self.name}` cannot be procced. {response.get('message')}")
return

if response.get(self.data_field) is None:
Expand Down Expand Up @@ -713,7 +710,6 @@ class EngagementStream(Stream):
Docs: https://legacydocs.hubspot.com/docs/methods/engagements/get-all-engagements
"""

entity = "engagement"
url = "/engagements/v1/engagements/paged"
more_key = "hasMore"
limit = 250
Expand Down
164 changes: 164 additions & 0 deletions airbyte-integrations/connectors/source-hubspot/source_hubspot/schemas/engagements.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,12 @@
"id": {
"type": ["null", "integer"]
},
"uid": {
"type": ["null", "string"]
},
"teamId": {
"type": ["null", "integer"]
},
"portalId": {
"type": ["null", "integer"]
},
Expand All @@ -14,6 +20,12 @@
"createdAt": {
"type": ["null", "integer"]
},
"createdBy": {
"type": ["null", "integer"]
},
"modifiedBy": {
"type": ["null", "integer"]
},
"lastUpdated": {
"type": ["null", "integer"]
},
Expand All @@ -26,6 +38,33 @@
"timestamp": {
"type": ["null", "integer"]
},
"bodyPreview": {
"type": ["null", "string"]
},
"bodyPreviewHtml": {
"type": ["null", "string"]
},
"bodyPreviewIsTruncated": {
"type": ["null", "boolean"]
},
"allAccessibleTeamIds": {
"type": ["null", "array"],
"items": {
"type": ["null", "integer"]
}
},
"activityType": {
"type": ["null", "string"]
},
"gdprDeleted": {
"type": ["null", "boolean"]
},
"source": {
"type": ["null", "string"]
},
"sourceId": {
"type": ["null", "string"]
},
"associations": {
"type": ["null", "object"],
"properties": {
Expand All @@ -46,6 +85,24 @@
"items": {
"type": ["null", "integer"]
}
},
"ownerIds": {
"type": ["null", "array"],
"items": {
"type": ["null", "integer"]
}
},
"workflowIds": {
"type": ["null", "array"],
"items": {
"type": ["null", "integer"]
}
},
"ticketIds": {
"type": ["null", "array"],
"items": {
"type": ["null", "integer"]
}
}
}
},
Expand Down Expand Up @@ -77,6 +134,17 @@
},
"lastName": {
"type": ["null", "string"]
},
"raw": {
"type": ["null", "string"]
}
}
},
"sender": {
"type": ["null", "object"],
"properties": {
"email": {
"type": ["null", "string"]
}
}
},
Expand All @@ -87,6 +155,15 @@
"properties": {
"email": {
"type": ["null", "string"]
},
"firstName": {
"type": ["null", "string"]
},
"lastName": {
"type": ["null", "string"]
},
"raw": {
"type": ["null", "string"]
}
}
}
Expand All @@ -98,6 +175,15 @@
"properties": {
"email": {
"type": ["null", "string"]
},
"firstName": {
"type": ["null", "string"]
},
"lastName": {
"type": ["null", "string"]
},
"raw": {
"type": ["null", "string"]
}
}
}
Expand Down Expand Up @@ -157,6 +243,84 @@
},
"disposition": {
"type": ["null", "string"]
},
"completionDate": {
"type": ["null", "integer"]
},
"taskType": {
"type": ["null", "string"]
},
"reminders": {
"type": ["null", "array"],
"items": {
"type": ["null", "integer"]
}
},
"threadId": {
"type": ["null", "string", "integer"]
},
"messageId": {
"type": ["null", "string"]
},
"loggedFrom": {
"type": ["null", "string"]
},
"attachedVideoOpened": {
"type": ["null", "boolean"]
},
"attachedVideoWatched": {
"type": ["null", "boolean"]
},
"trackerKey": {
"type": ["null", "string"]
},
"sendDefaultReminder": {
"type": ["null", "boolean"]
},
"source": {
"type": ["null", "string"]
},
"unknownVisitorConversation": {
"type": ["null", "boolean"]
},
"facsimileSendId": {
"type": ["null", "string"]
},
"sentVia": {
"type": ["null", "string"]
},
"sequenceStepOrder": {
"type": ["null", "integer"]
},
"externalUrl": {
"type": ["null", "string"]
},
"postSendStatus": {
"type": ["null", "string"]
},
"errorMessage": {
"type": ["null", "string"]
},
"recipientDropReasons": {
"type": ["null", "string"]
},
"calleeObjectId": {
"type": ["null", "integer"]
},
"calleeObjectType": {
"type": ["null", "string"]
},
"mediaProcessingStatus": {
"type": ["null", "string"]
},
"sourceId": {
"type": ["null", "string"]
},
"priority": {
"type": ["null", "string"]
},
"isAllDay": {
"type": ["null", "boolean"]
}
}
}
Expand Down
14 changes: 7 additions & 7 deletions docs/integrations/sources/hubspot.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,15 +37,15 @@ This source is capable of syncing the following tables and their data:
* [Workflows](https://legacydocs.hubspot.com/docs/methods/workflows/v3/get_workflows)

### A note on the `engagements` stream
Objects in the `engagements` stream can have one of the following types: `note`, `email`, `task`, `meeting`, `call`.
Objects in the `engagements` stream can have one of the following types: `note`, `email`, `task`, `meeting`, `call`.

Depending on the type of engagement, different properties will be set for that object in the `engagements_metadata` table in the destination.
Depending on the type of engagement, different properties will be set for that object in the `engagements_metadata` table in the destination.

* A `call` engagement will have a corresponding `engagements_metadata` object with non-null values in the `toNumber`, `fromNumber`, `status`, `externalId`, `durationMilliseconds`, `externalAccountId`, `recordingUrl`, `body`, and `disposition` columns.
* An `email` engagement will have a corresponding `engagements_metadata` object with with non-null values in the `subject`, `html`, and `text` columns. In addition, there will be records in four related tables, `engagements_metadata_from`, `engagements_metadata_to`, `engagements_metadata_cc`, `engagements_metadata_bcc`.
* A `meeting` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `startTime`, `endTime`, and `title` columns.
* A `note` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body` column.
* A `task` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `status`, and `forObjectType` columns.
* An `email` engagement will have a corresponding `engagements_metadata` object with with non-null values in the `subject`, `html`, and `text` columns. In addition, there will be records in four related tables, `engagements_metadata_from`, `engagements_metadata_to`, `engagements_metadata_cc`, `engagements_metadata_bcc`.
* A `meeting` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `startTime`, `endTime`, and `title` columns.
* A `note` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body` column.
* A `task` engagement will have a corresponding `engagements_metadata` object with non-null values in the `body`, `status`, and `forObjectType` columns.


**Note**: HubSpot API currently only supports `quotes` endpoint using API Key, using Oauth it is impossible to access this stream (as reported by [community.hubspot.com](https://community.hubspot.com/t5/APIs-Integrations/Help-with-using-Feedback-CRM-API-and-Quotes-CRM-API/m-p/449104/highlight/true#M44411)).
Expand Down Expand Up @@ -110,6 +110,7 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt

| Version | Date | Pull Request | Subject |
|:--------|:-----------| :--- |:-----------------------------------------------------------------------------------------------------------------------------------------------|
| 0.1.31 | 2022-01-11 | [9385](https://github.com/airbytehq/airbyte/pull/9385) | Remove auto-generated `properties` from `Engagements` stream |
| 0.1.30 | 2021-01-10 | [9129](https://github.com/airbytehq/airbyte/pull/9129) | Created Contacts list memberships streams |
| 0.1.29 | 2021-12-17 | [8699](https://github.com/airbytehq/airbyte/pull/8699) | Add incremental sync support for `companies`, `contact_lists`, `contacts`, `deals`, `line_items`, `products`, `quotes`, `tickets` streams |
| 0.1.28 | 2021-12-15 | [8429](https://github.com/airbytehq/airbyte/pull/8429) | Update fields and descriptions |
Expand All @@ -134,4 +135,3 @@ If you are using Oauth, most of the streams require the appropriate [scopes](htt
| 0.1.9 | 2021-08-11 | [5334](https://github.com/airbytehq/airbyte/pull/5334) | Fix empty strings inside float datatype |
| 0.1.8 | 2021-08-06 | [5250](https://github.com/airbytehq/airbyte/pull/5250) | Fix issue with printing exceptions |
| 0.1.7 | 2021-07-27 | [4913](https://github.com/airbytehq/airbyte/pull/4913) | Update fields schema |

0 comments on commit 9909cd6

Please sign in to comment.