Feature/add union data

CHANGELOG.md

@@ -1,3 +1,7 @@
+# dbt_hubspot_source v0.15.0
+
+Unioning coming soon...
+
 # dbt_hubspot_source v0.14.0
 [PR #122](https://github.com/fivetran/dbt_hubspot_source/pull/122) includes the following updates:
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'hubspot_source'
-version: '0.14.0'
+version: '0.15.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
  pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
  dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
  port: 5439
- schema: hubspot_source_integration_tests_999
+ schema: hubspot_source_integration_tests_001
  threads: 8
  bigquery:
  type: bigquery
  method: service-account-json
  project: 'dbt-package-testing'
- schema: hubspot_source_integration_tests_999
+ schema: hubspot_source_integration_tests_001
  threads: 8
  keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
  snowflake:
@@ -33,7 +33,7 @@ integration_tests:
  role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
  database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
  warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
- schema: hubspot_source_integration_tests_999
+ schema: hubspot_source_integration_tests_001
  threads: 8
  postgres:
  type: postgres
@@ -42,13 +42,13 @@ integration_tests:
  pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
  dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
  port: 5432
- schema: hubspot_source_integration_tests_999
+ schema: hubspot_source_integration_tests_001
  threads: 8
  databricks:
  catalog: null
  host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
  http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
- schema: hubspot_source_integration_tests_999
+ schema: hubspot_source_integration_tests_001
  threads: 8
  token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
  type: databricks

integration_tests/dbt_project.yml

@@ -1,12 +1,12 @@
 name: 'hubspot_source_integration_tests'
-version: '0.14.0'
+version: '0.15.0'
 profile: 'integration_tests'
 config-version: 2
 models:
  hubspot_source:
  +schema:
 vars:
- hubspot_schema: hubspot_source_integration_tests_999
+ hubspot_schema: hubspot_source_integration_tests_001
  hubspot_source:
  hubspot_service_enabled: true
  # hubspot_sales_enabled: true # enable when generating docs

models/docs.md

@@ -2,6 +2,10 @@
 Timestamp of when Fivetran synced a record.
 {% enddocs %}
 
+{% docs source_relation %}
+The schema or database this record came from if you are unioning multiple connectors together in this package. If you are running the package on a single connector, this will be its schema name.
+{% enddocs %}
+
 {% docs _fivetran_deleted %}
 Boolean indicating whether a record has been deleted in Hubspot and/or inferred deleted in Hubspot by Fivetran; _fivetran_deleted and is_deleted fields are equivalent. 
 {% enddocs %}

models/stg_hubspot__company.sql

@@ -14,6 +14,14 @@ with base as (
  staging_columns=get_company_columns()
  )
  }}
+
+ {{ 
+ fivetran_utils.source_relation(
+ union_schema_variable='hubspot_union_schemas', 
+ union_database_variable='hubspot_union_databases'
+ ) 
+ }}
+
  from base
 
 ), fields as (
@@ -27,6 +35,14 @@ with base as (
  staging_columns=get_company_columns()
  )
  }}
+
+ {{ 
+ fivetran_utils.source_relation(
+ union_schema_variable='hubspot_union_schemas', 
+ union_database_variable='hubspot_union_databases'
+ ) 
+ }}
+
  {% if all_passthrough_column_check('stg_hubspot__company_tmp',get_company_columns()) > 0 %}
  -- just pass everything through if extra columns are present, but ensure required columns are present.
  ,{{ 
@@ -52,7 +68,8 @@ with base as (
  city,
  state,
  country,
- company_annual_revenue
+ company_annual_revenue,
+ source_relation
 
  --The below macro adds the fields defined within your hubspot__ticket_pass_through_columns variable into the staging model
  {{ fivetran_utils.fill_pass_through_columns('hubspot__company_pass_through_columns') }}

models/stg_hubspot__company.yml

@@ -19,9 +19,16 @@ models:
  description: '{{ doc("history_name") }}'
  - name: new_value
  description: '{{ doc("history_value") }}'
+ - name: source_relation
+ description: '{{ doc("source_relation") }}'
 
  - name: stg_hubspot__company
  description: Each record represents a company in Hubspot.
+ tests:
+ - dbt_utils.unique_combination_of_columns:
+ combination_of_columns: 
+ - company_id
+ - source_relation 
  columns:
  - name: _fivetran_synced
  description: '{{ doc("_fivetran_synced") }}'
@@ -30,7 +37,6 @@ models:
  - name: company_id
  description: The ID of the company.
  tests:
- - unique
  - not_null
  - name: company_name
  description: The name of the company.
@@ -52,3 +58,5 @@ models:
  description: The country where the company is located.
  - name: company_annual_revenue
  description: The actual or estimated annual revenue of the company.
+ - name: source_relation
+ description: '{{ doc("source_relation") }}'

models/stg_hubspot__company_property_history.sql

@@ -14,6 +14,14 @@ with base as (
  staging_columns=get_company_property_history_columns()
  )
  }}
+
+ {{ 
+ fivetran_utils.source_relation(
+ union_schema_variable='hubspot_union_schemas', 
+ union_database_variable='hubspot_union_databases'
+ ) 
+ }}
+
  from base
 
 ), fields as (
@@ -25,7 +33,8 @@ with base as (
  source as change_source,
  source_id as change_source_id,
  cast(change_timestamp as {{ dbt.type_timestamp() }}) as change_timestamp, -- source field name = timestamp ; alias declared in macros/get_company_property_history_columns.sql
- value as new_value
+ value as new_value,
+ source_relation
  from macro
 
 )

models/stg_hubspot__contact.sql

@@ -14,6 +14,14 @@ with base as (
  staging_columns=get_contact_columns()
  )
  }}
+
+ {{ 
+ fivetran_utils.source_relation(
+ union_schema_variable='hubspot_union_schemas', 
+ union_database_variable='hubspot_union_databases'
+ ) 
+ }}
+
  from base
 
 ), fields as (
@@ -27,6 +35,14 @@ with base as (
  staging_columns=get_contact_columns()
  )
  }}
+
+ {{ 
+ fivetran_utils.source_relation(
+ union_schema_variable='hubspot_union_schemas', 
+ union_database_variable='hubspot_union_databases'
+ ) 
+ }}
+
  {% if all_passthrough_column_check('stg_hubspot__contact_tmp',get_contact_columns()) > 0 %}
  -- just pass everything through if extra columns are present, but ensure required columns are present.
  ,{{ 
@@ -50,7 +66,8 @@ with base as (
  cast(created_date as {{ dbt.type_timestamp() }}) as created_date,
  job_title,
  company_annual_revenue,
- cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced
+ cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced,
+ source_relation
 
  --The below macro adds the fields defined within your hubspot__contact_pass_through_columns variable into the staging model
  {{ fivetran_utils.fill_pass_through_columns('hubspot__contact_pass_through_columns') }}

models/stg_hubspot__contact.yml

@@ -14,9 +14,16 @@ models:
  description: The ID of the related contact.
  - name: contact_list_id
  description: The ID of the related contact list.
+ - name: source_relation
+ description: '{{ doc("source_relation") }}'
 
  - name: stg_hubspot__contact_list
  description: Each record represents a contact list in Hubspot.
+ tests:
+ - dbt_utils.unique_combination_of_columns:
+ combination_of_columns: 
+ - contact_list_id
+ - source_relation 
  columns:
  - name: is_contact_list_deleted
  description: '{{ doc("is_deleted") }}'
@@ -25,7 +32,6 @@ models:
  - name: contact_list_id
  description: The ID of the contact list.
  tests:
- - unique
  - not_null
  - name: contact_list_name
  description: The name of the contact list.
@@ -51,6 +57,8 @@ models:
  description: '{{ doc("portal_id") }}'
  - name: updated_timestamp
  description: A timestamp of the time that the list was last modified.
+ - name: source_relation
+ description: '{{ doc("source_relation") }}'
 
  - name: stg_hubspot__contact_property_history
  description: Each record represents a change to contact record in Hubspot.
@@ -69,6 +77,8 @@ models:
  description: '{{ doc("history_name") }}'
  - name: new_value
  description: '{{ doc("history_value") }}'
+ - name: source_relation
+ description: '{{ doc("source_relation") }}'
 
  - name: stg_hubspot__contact_merge_audit
  description: Each record contains a contact merge event and the contacts effected by the merge.
@@ -93,9 +103,16 @@ models:
  description: The ID of the user.
  - name: vid_to_merge
  description: The ID of the contact which was merged.
+ - name: source_relation
+ description: '{{ doc("source_relation") }}'
 
  - name: stg_hubspot__contact
  description: Each record represents a contact in Hubspot.
+ tests:
+ - dbt_utils.unique_combination_of_columns:
+ combination_of_columns: 
+ - contact_id
+ - source_relation
  columns:
  - name: is_contact_deleted
  description: '{{ doc("is_deleted") }}'
@@ -104,7 +121,6 @@ models:
  - name: contact_id
  description: The ID of the contact.
  tests:
- - unique
  - not_null
  - name: email
  description: The email address of the contact.
@@ -126,4 +142,6 @@ models:
  description: >
  List of mappings representing contact IDs that have been merged into the contact at hand.
  Format: <merged_contact_id>:<merged_at_in_epoch_time>;<second_merged_contact_id>:<merged_at_in_epoch_time>
- This field has replaced the `CONTACT_MERGE_AUDIT` table, which was deprecated by the Hubspot v3 CRM API.
+ This field has replaced the `CONTACT_MERGE_AUDIT` table, which was deprecated by the Hubspot v3 CRM API.
+ - name: source_relation
+ description: '{{ doc("source_relation") }}'

models/stg_hubspot__contact_list.sql

@@ -14,6 +14,14 @@ with base as (
  staging_columns=get_contact_list_columns()
  )
  }}
+
+ {{ 
+ fivetran_utils.source_relation(
+ union_schema_variable='hubspot_union_schemas', 
+ union_database_variable='hubspot_union_databases'
+ ) 
+ }}
+
  from base
 
 ), fields as (
@@ -32,7 +40,8 @@ with base as (
  metadata_size,
  name as contact_list_name,
  portal_id,
- cast(updated_at as {{ dbt.type_timestamp() }}) as updated_timestamp
+ cast(updated_at as {{ dbt.type_timestamp() }}) as updated_timestamp,
+ source_relation
  from macro
 
 )

models/stg_hubspot__contact_list_member.sql

@@ -14,6 +14,14 @@ with base as (
  staging_columns=get_contact_list_member_columns()
  )
  }}
+
+ {{ 
+ fivetran_utils.source_relation(
+ union_schema_variable='hubspot_union_schemas', 
+ union_database_variable='hubspot_union_databases'
+ ) 
+ }}
+
  from base
 
 ), fields as (
@@ -23,7 +31,8 @@ with base as (
  cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced,
  cast(added_at as {{ dbt.type_timestamp() }}) as added_timestamp,
  contact_id,
- contact_list_id
+ contact_list_id,
+ source_relation
  from macro
 
 )

models/stg_hubspot__contact_merge_audit.sql

@@ -14,6 +14,14 @@ with base as (
  staging_columns=get_contact_merge_audit_columns()
  )
  }}
+
+ {{ 
+ fivetran_utils.source_relation(
+ union_schema_variable='hubspot_union_schemas', 
+ union_database_variable='hubspot_union_databases'
+ ) 
+ }}
+
  from base
 
 ), fields as (
@@ -33,7 +41,8 @@ with base as (
  as {{ dbt.type_timestamp() }}) as timestamp_at,
  user_id,
  vid_to_merge,
- cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced
+ cast(_fivetran_synced as {{ dbt.type_timestamp() }}) as _fivetran_synced,
+ source_relation
  from macro
 
 )

models/stg_hubspot__contact_property_history.sql

@@ -14,6 +14,14 @@ with base as (
  staging_columns=get_contact_property_history_columns()
  )
  }}
+
+ {{ 
+ fivetran_utils.source_relation(
+ union_schema_variable='hubspot_union_schemas', 
+ union_database_variable='hubspot_union_databases'
+ ) 
+ }}
+
  from base
 
 ), fields as (
@@ -25,7 +33,9 @@ with base as (
  source as change_source,
  source_id as change_source_id,
  cast(change_timestamp as {{ dbt.type_timestamp() }}) as change_timestamp, -- source field name = timestamp ; alias declared in macros/get_contact_property_history_columns.sql
- value as new_value
+ value as new_value,
+ source_relation
+
  from macro
 
 )