bug/renamed-columns (#55)

* bug/renamed-columns

* regen docs

* update comments

* Update CHANGELOG.md

Co-authored-by: Avinash Kunnath <[email protected]>

* update changelog and bump packages

---------

Co-authored-by: Avinash Kunnath <[email protected]>

CHANGELOG.md

@@ -1,3 +1,16 @@
+# dbt_salesforce v1.1.0
+[PR #55](https://github.com/fivetran/dbt_salesforce/pull/55) includes the following updates:
+
+## 🚨 Breaking Change 🚨
+- This change is made breaking due to changes made in the source package. See the [v1.1.0 dbt_salesforce_source release notes](https://github.com/fivetran/dbt_salesforce_source/releases/tag/v1.1.0) for more details.
+- Added logic to support user-specified scenarios where the Fivetran Salesforce connector syncs column names using the original Salesforce API naming convention. For example, while Fivetran typically provides the column as `created_date`, some users might choose to receive it as `CreatedDate` according to the API naming. This update ensures the package is automatically compatible with both naming conventions.
+  - Specifically, the package now performs a COALESCE, preferring the original Salesforce API naming. If the original naming is not present, the Fivetran version is used instead.
+  - Renamed columns are now explicitly cast to prevent conflicts during the COALESCE. 
+  - ❗This change is considered breaking since the resulting column types may differ from prior versions of this package.
+
+## Under the hood
+- Added validation test to ensure the final column names generated before and after this update remain the same.
+
 # dbt_salesforce v1.0.2
 [PR #52](https://github.com/fivetran/dbt_salesforce/pull/52) includes the following updates:
 

README.md

@@ -75,7 +75,7 @@ Include the following salesforce package version in your `packages.yml`
 ```yaml
 packages:
   - package: fivetran/salesforce 
-    version: [">=1.0.0", "<1.1.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=1.1.0", "<1.2.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 Do NOT include the `salesforce_source` package in this file. The transformation package itself has a dependency on it and will install the source package as well.
@@ -287,7 +287,7 @@ This dbt package is dependent on the following dbt packages. For more informatio
 ```yml
 packages:
     - package: fivetran/salesforce_source
-      version: [">=1.0.0", "<1.1.0"]
+      version: [">=1.1.0", "<1.2.0"]
     - package: fivetran/fivetran_utils
       version: [">=0.4.0", "<0.5.0"]
     - package: dbt-labs/dbt_utils

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'salesforce'
-version: '1.0.2'
+version: '1.1.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   salesforce:

integration_tests/dbt_project.yml

@@ -1,13 +1,12 @@
 name: 'salesforce_integration_tests'
-version: '1.0.2'
+version: '1.1.0'
 config-version: 2
 
 profile: 'integration_tests'
 
 models:
-  salesforce:
-    materialized: table
-    +schema: "salesforce_{{ var('directed_schema','dev') }}"
+  +materialized: table
+  +schema: "salesforce_{{ var('directed_schema','dev') }}"
 
 vars:
   # enable history models when generating docs!

integration_tests/tests/consistency/consistency_columns.sql

@@ -0,0 +1,60 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+/* This test is to make sure the final columns produced are the same between versions.
+Only one test is needed since it will fetch all tables and all columns in each schema.
+!!! THIS TEST IS WRITTEN FOR BIGQUERY!!! */
+{% if target.type == 'bigquery' %}
+with prod as (
+    select
+        table_name,
+        column_name,
+        data_type
+    from {{ target.schema }}_salesforce_prod.INFORMATION_SCHEMA.COLUMNS
+),
+
+dev as (
+    select
+        table_name,
+        column_name,
+        data_type
+    from {{ target.schema }}_salesforce_dev.INFORMATION_SCHEMA.COLUMNS
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final
+
+{% else %}
+{{ print('This is written to run on bigquery. If you need to run on another warehouse, add another version for that warehouse') }}
+
+{% endif %}

integration_tests/tests/consistency/consistency_daily_activity.sql

@@ -7,25 +7,41 @@
 with prod as (
     select *
     from {{ target.schema }}_salesforce_prod.salesforce__daily_activity
+    where date(date_day) < date({{ dbt.current_timestamp() }})
 ),
 
 dev as (
     select *
     from {{ target.schema }}_salesforce_dev.salesforce__daily_activity
+    where date(date_day) < date({{ dbt.current_timestamp() }})
 ),
 
-final as (
-    -- test will fail if any rows from prod are not found in dev
-    (select * from prod
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
     except distinct
-    select * from dev)
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
 
     union all -- union since we only care if rows are produced
 
-    -- test will fail if any rows from dev are not found in prod
-    (select * from dev
-    except distinct
-    select * from prod)
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
 )
 
 select *

packages.yml

@@ -1,3 +1,3 @@
 packages:
 - package: fivetran/salesforce_source
-  version: [">=1.0.0", "<1.1.0"]
+  version: [">=1.1.0", "<1.2.0"]