chore(data-warehouse): add more hubspot fields (#20115)

* add fields

* Update UI snapshots for `chromium` (2)

* Update UI snapshots for `chromium` (2)

* don't sync newly added default props

* update logic for activity start

---------

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

frontend/src/scenes/data-warehouse/external/sourceModalLogic.tsx

@@ -220,6 +220,7 @@ export const sourceModalLogic = kea<sourceModalLogicType>([
                         'crm.objects.deals.read',
                         'tickets',
                         'crm.objects.quotes.read',
+                        'sales-email-read',
                     ]
 
                     const params = new URLSearchParams()

posthog/temporal/data_imports/external_data_job.py

@@ -11,7 +11,6 @@
 from posthog.temporal.batch_exports.base import PostHogWorkflow
 
 from posthog.warehouse.data_load.validate_schema import validate_schema_and_update_table
-from posthog.temporal.data_imports.pipelines.schemas import PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING
 from posthog.temporal.data_imports.pipelines.pipeline import DataImportPipeline, PipelineInputs
 from posthog.warehouse.external_data_source.jobs import (
     create_external_data_job,
@@ -61,14 +60,11 @@ async def create_external_data_job_model(inputs: CreateExternalDataJobInputs) ->
         schemas_to_sync = await sync_to_async(get_postgres_schemas)(  # type: ignore
             host, port, database, user, password, schema
         )
-    else:
-        schemas_to_sync = list(PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING[source.source_type])
-
-    await sync_to_async(sync_old_schemas_with_new_schemas)(  # type: ignore
-        schemas_to_sync,
-        source_id=inputs.external_data_source_id,
-        team_id=inputs.team_id,
-    )
+        await sync_to_async(sync_old_schemas_with_new_schemas)(  # type: ignore
+            schemas_to_sync,
+            source_id=inputs.external_data_source_id,
+            team_id=inputs.team_id,
+        )
 
     schemas = await sync_to_async(get_active_schemas_for_source_id)(  # type: ignore
         team_id=inputs.team_id, source_id=inputs.external_data_source_id

posthog/temporal/data_imports/pipelines/hubspot/settings.py

@@ -9,19 +9,26 @@
 DEAL = "deal"
 TICKET = "ticket"
 QUOTE = "quote"
+EMAILS = "emails"
+MEETINGS = "meetings"
 
 CRM_CONTACTS_ENDPOINT = "/crm/v3/objects/contacts?associations=deals,tickets,quotes"
 CRM_COMPANIES_ENDPOINT = "/crm/v3/objects/companies?associations=contacts,deals,tickets,quotes"
 CRM_DEALS_ENDPOINT = "/crm/v3/objects/deals"
 CRM_TICKETS_ENDPOINT = "/crm/v3/objects/tickets"
 CRM_QUOTES_ENDPOINT = "/crm/v3/objects/quotes"
+CRM_EMAILS_ENDPOINT = "/crm/v3/objects/emails"
+CRM_MEETINGS_ENDPOINT = "/crm/v3/objects/meetings"
+
 
 CRM_OBJECT_ENDPOINTS = {
     CONTACT: CRM_CONTACTS_ENDPOINT,
     COMPANY: CRM_COMPANIES_ENDPOINT,
     DEAL: CRM_DEALS_ENDPOINT,
     TICKET: CRM_TICKETS_ENDPOINT,
     QUOTE: CRM_QUOTES_ENDPOINT,
+    EMAILS: CRM_EMAILS_ENDPOINT,
+    MEETINGS: CRM_MEETINGS_ENDPOINT,
 }
 
 WEB_ANALYTICS_EVENTS_ENDPOINT = "/events/v3/events?objectType={objectType}&objectId={objectId}&occurredAfter={occurredAfter}&occurredBefore={occurredBefore}&sort=-occurredAt"
@@ -32,6 +39,8 @@
     "deals": DEAL,
     "tickets": TICKET,
     "quotes": QUOTE,
+    "emails": EMAILS,
+    "meetings": MEETINGS,
 }
 
 OBJECT_TYPE_PLURAL = {v: k for k, v in OBJECT_TYPE_SINGULAR.items()}
@@ -43,6 +52,8 @@
     OBJECT_TYPE_PLURAL[COMPANY],
     OBJECT_TYPE_PLURAL[TICKET],
     OBJECT_TYPE_PLURAL[QUOTE],
+    OBJECT_TYPE_PLURAL[EMAILS],
+    OBJECT_TYPE_PLURAL[MEETINGS],
 )
 
 DEFAULT_DEAL_PROPS = [
@@ -95,12 +106,39 @@
     "hs_title",
 ]
 
+DEFAULT_EMAIL_PROPS = [
+    "hs_timestamp",
+    "hs_email_direction",
+    "hs_email_html",
+    "hs_email_status",
+    "hs_email_subject",
+    "hs_email_text",
+    "hs_attachment_ids",
+    "hs_email_headers",
+]
+
+DEFAULT_MEETINGS_PROPS = [
+    "hs_timestamp",
+    "hs_meeting_title",
+    "hs_meeting_body",
+    "hs_internal_meeting_notes",
+    "hs_meeting_external_URL",
+    "hs_meeting_location",
+    "hs_meeting_start_time",
+    "hs_meeting_end_time",
+    "hs_meeting_outcome",
+    "hs_activity_type",
+    "hs_attachment_ids",
+]
+
 DEFAULT_PROPS = {
     OBJECT_TYPE_PLURAL[CONTACT]: DEFAULT_CONTACT_PROPS,
     OBJECT_TYPE_PLURAL[COMPANY]: DEFAULT_COMPANY_PROPS,
     OBJECT_TYPE_PLURAL[DEAL]: DEFAULT_DEAL_PROPS,
     OBJECT_TYPE_PLURAL[TICKET]: DEFAULT_TICKET_PROPS,
     OBJECT_TYPE_PLURAL[QUOTE]: DEFAULT_QUOTE_PROPS,
+    OBJECT_TYPE_PLURAL[EMAILS]: DEFAULT_EMAIL_PROPS,
+    OBJECT_TYPE_PLURAL[MEETINGS]: DEFAULT_MEETINGS_PROPS,
 }
 
 ALL = ("ALL",)

posthog/temporal/tests/external_data/test_external_data_job.py

@@ -140,8 +140,6 @@ async def test_create_external_job_activity(activity_environment, team, **kwargs
     runs = ExternalDataJob.objects.filter(id=run_id)
     assert await sync_to_async(runs.exists)()  # type:ignore
     assert len(schemas) == 0
-    count = await sync_to_async(ExternalDataSchema.objects.filter(source_id=new_source.pk).count)()  # type:ignore
-    assert count == len(PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING[new_source.source_type])
 
 
 @pytest.mark.django_db(transaction=True)
@@ -176,9 +174,6 @@ async def test_create_external_job_activity_schemas_exist(activity_environment,
     runs = ExternalDataJob.objects.filter(id=run_id)
     assert await sync_to_async(runs.exists)()  # type:ignore
     assert len(schemas) == 1
-    # doesn't overlap
-    count = await sync_to_async(ExternalDataSchema.objects.filter(source_id=new_source.pk).count)()  # type:ignore
-    assert count == len(PIPELINE_TYPE_SCHEMA_DEFAULT_MAPPING[new_source.source_type])
 
 
 @pytest.mark.django_db(transaction=True)