Merge pull request #74 from fivetran/bugfix/consolidate-exch-account-…

…book-add

Bugfix/consolidate exch account book add

CHANGELOG.md

@@ -1,3 +1,13 @@
+# dbt_netsuite v0.9.0
+
+[PR #74](https://github.com/fivetran/dbt_netsuite/pull/74)includes the following updates:
+## 🚨 Breaking Changes 🚨
+- Removed the `int_netsuite2__consolidated_exchange_rates` model 
+ - Originally the `accounting_book_id` field was brought into the `int_netsuite2__acctxperiod_exchange_rate_map` model via `int_netsuite2__consolidated_exchange_rates`, but this was resulting in duplicate records downstream in the `netsuite2__transaction_details` model due to the way it was being joined. Now we have brought in `accounting_book_id` (accountingbook) via the `stg_netsuite2__consolidated_exchange_rates` model, so we do not have a need for `int_netsuite2__consolidated_exchange_rates` 
+
+## Test Updates
+- Added `account_id` to the unique combination test for `netsuite2__balance_sheet`
+
 # dbt_netsuite v0.8.1
 [PR #73](https://github.com/fivetran/dbt_netsuite/pull/73) applies the following changes:
 
@@ -15,7 +25,6 @@ vars:
 ## 🔧 Under the Hood 🔩
 - Incorporated the new `fivetran_utils.drop_schemas_automation` macro into the end of each Buildkite integration test job.
 - Updated the pull request [templates](/.github).
-
 # dbt_netsuite v0.8.0
 [PR #66](https://github.com/fivetran/dbt_netsuite/pull/66) applies the following changes:
 

README.md

@@ -100,7 +100,7 @@ Include the following netsuite package version in your `packages.yml` file:
 ```yaml
 packages:
  - package: fivetran/netsuite
- version: [">=0.8.0", "<0.9.0"]
+ version: [">=0.9.0", "<0.10.0"]
 ```
 ## Step 3: Define Netsuite.com or Netsuite2 Source
 As of April 2022 Fivetran made available a new Netsuite connector which leverages the Netsuite2 endpoint opposed to the original Netsuite.com endpoint. This package is designed to run for either or, not both. By default the `netsuite_data_model` variable for this package is set to the original `netsuite` value which runs the netsuite.com version of the package. If you would like to run the package on Netsuite2 data, you may adjust the `netsuite_data_model` variable to run the `netsuite2` version of the package.
@@ -223,7 +223,7 @@ This dbt package is dependent on the following dbt packages. Please be aware tha
 ```yml
 packages:
  - package: fivetran/netsuite_source
- version: [">=0.6.0", "<0.7.0"]
+ version: [">=0.7.0", "<0.8.0"]
 
  - package: fivetran/fivetran_utils
  version: [">=0.4.0", "<0.5.0"]

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'netsuite'
-version: '0.8.1'
+version: '0.9.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
  pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
  dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
  port: 5439
- schema: netsuite_integration_tests_1
+ schema: netsuite_integration_tests_3
  threads: 8
  bigquery:
  type: bigquery
  method: service-account-json
  project: 'dbt-package-testing'
- schema: netsuite_integration_tests_1
+ schema: netsuite_integration_tests_3
  threads: 8
  keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
  snowflake:
@@ -33,7 +33,7 @@ integration_tests:
  role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
  database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
  warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
- schema: netsuite_integration_tests_1
+ schema: netsuite_integration_tests_3
  threads: 8
  postgres:
  type: postgres
@@ -42,13 +42,13 @@ integration_tests:
  pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
  dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
  port: 5432
- schema: netsuite_integration_tests_1
+ schema: netsuite_integration_tests_3
  threads: 8
  databricks:
  catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
  host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
  http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
- schema: netsuite_integration_tests_1
+ schema: netsuite_integration_tests_3
  threads: 2
  token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
  type: databricks

integration_tests/dbt_project.yml

@@ -1,11 +1,11 @@
 name: 'netsuite_integration_tests'
-version: '0.8.1'
+version: '0.9.0'
 profile: 'integration_tests'
 config-version: 2
 models:
  +materialized: table
 vars:
- netsuite_schema: netsuite_integration_tests_1
+ netsuite_schema: netsuite_integration_tests_3
  netsuite_data_model_override: netsuite
  netsuite_source:
  # Netsuite Seed Data

integration_tests/seeds/netsuite2_accounting_period_data.csv

@@ -1,6 +1,6 @@
 id,_fivetran_deleted,_fivetran_synced,alllocked,allownonglchanges,aplocked,arlocked,closed,closedondate,date_deleted,enddate,isadjust,isinactive,isposting,isquarter,isyear,lastmodifieddate,parent,periodname,startdate
-5,FALSE,2022-07-10 23:00:33,T,F,T,T,T,2002-03-31 0:00:00,,2002-03-31 0:00:00,F,F,T,F,F,2019-02-20 12:32:15,2,Mar 2002,2002-03-01 00:00:00
-8,FALSE,2022-07-10 23:00:33,T,F,T,T,T,2002-05-31 0:00:00,,2002-05-31 0:00:00,F,F,T,F,F,2019-02-20 12:32:15,6,May 2002,2002-05-01 00:00:00
-9,FALSE,2022-07-10 23:00:33,T,F,T,T,T,2002-06-30 0:00:00,,2002-06-30 0:00:00,F,F,T,F,F,2019-02-20 12:32:15,6,Jun 2002,2002-06-01 00:00:00
-21,FALSE,2022-07-10 23:00:33,T,F,T,T,T,2001-02-28 0:00:00,,2001-02-28 0:00:00,F,F,T,F,F,2019-02-20 12:32:15,19,Feb 2001,2001-02-01 00:00:00
-20,FALSE,2022-07-10 23:00:33,T,F,T,T,T,2001-01-31 0:00:00,,2001-01-31 0:00:00,F,F,T,F,F,2019-02-20 12:32:15,19,Jan 2001,2001-01-01 00:00:00
+5,FALSE,2022-07-10 23:00:33,T,F,T,T,T,2002-03-31 0:00:00,,2002-03-31 0:00:00,F,F,T,F,F,2019-02-20 12:32:15,2,Mar 2002,2002-03-01 0:00:00
+8,FALSE,2022-07-10 23:00:33,T,F,T,T,T,2002-05-31 0:00:00,,2002-05-31 0:00:00,F,F,T,F,F,2019-02-20 12:32:15,6,May 2002,2002-05-01 0:00:00
+9,FALSE,2022-07-10 23:00:33,T,F,T,T,T,2002-06-30 0:00:00,,2002-06-30 0:00:00,F,F,T,F,F,2019-02-20 12:32:15,6,Jun 2002,2002-06-01 0:00:00
+21,FALSE,2022-07-10 23:00:33,T,F,T,T,T,2001-02-28 0:00:00,,2001-02-28 0:00:00,F,F,T,F,F,2019-02-20 12:32:15,19,Feb 2001,2001-02-01 0:00:00
+20,FALSE,2022-07-10 23:00:33,T,F,T,T,T,2001-01-31 0:00:00,,2001-01-31 0:00:00,F,F,T,F,F,2019-02-20 12:32:15,19,Jan 2001,2001-01-01 0:00:00

integration_tests/seeds/netsuite2_consolidated_exchange_rate_data.csv

@@ -1,6 +1,6 @@
-id,_fivetran_deleted,_fivetran_synced,averagerate,currentrate,date_deleted,externalid,fromcurrency,fromsubsidiary,historicalrate,postingperiod,tocurrency,tosubsidiary
-1979,FALSE,2022-07-10 22:16:59,1,1,,,1,1,1,289,1,1
-1944,FALSE,2022-07-10 22:16:59,1,1,,,1,1,1,282,1,1
-1930,FALSE,2022-07-10 22:16:59,1,1,,,1,1,1,280,1,1
-1958,FALSE,2022-07-10 22:16:59,1,1,,,1,1,1,285,1,1
-1951,FALSE,2022-07-10 22:16:59,1,1,,,1,1,1,284,1,1
+id,_fivetran_deleted,_fivetran_synced,averagerate,currentrate,date_deleted,externalid,fromcurrency,fromsubsidiary,historicalrate,postingperiod,tocurrency,tosubsidiary,accountingbook
+1979,FALSE,2022-07-10 22:16:59,1,1,,,1,1,1,289,1,1,1
+1944,FALSE,2022-07-10 22:16:59,1,1,,,1,1,1,282,1,1,1
+1930,FALSE,2022-07-10 22:16:59,1,1,,,1,1,1,280,1,1,1
+1958,FALSE,2022-07-10 22:16:59,1,1,,,1,1,1,285,1,1,1
+1951,FALSE,2022-07-10 22:16:59,1,1,,,1,1,1,284,1,1,1

integration_tests/seeds/netsuite2_job_data.csv

@@ -1,5 +1,5 @@
 id,_fivetran_deleted,_fivetran_synced,accountnumber,actualtime,allowallresourcesfortasks,allowexpenses,allowtime,alternatecontact,altname,billingschedule,calculatedenddate,calculatedenddatebaseline,category,comments,companyname,contact,contactlist,currency,currencyprecision,custentity1,custentity4,custentity5,customer,date_deleted,datecreated,defaultbillingaddress,defaultshippingaddress,enddate,entityid,entitynumber,entitystatus,entitytitle,estimatedcost,estimatedgrossprofit,estimatedgrossprofitpercent,estimatedlaborcost,estimatedlaborcostbaseline,estimatedlaborrevenue,estimatedrevenue,estimatedtime,estimatedtimeoverride,estimatedtimeoverridebaseline,externalid,files,forecastchargerunondemand,fxrate,includecrmtasksintotals,isexempttime,isinactive,isproductivetime,isutilizedtime,jobbillingtype,jobitem,jobprice,jobtype,lastbaselinedate,lastmodifieddate,limittimetoassignees,materializetime,parent,percentcomplete,percenttimecomplete,primarycontact,projectedenddate,projectedenddatebaseline,projectmanager,startdate,startdatebaseline,timeapproval,timeremaining
-1548,FALSE,2022-05-27 13:12:02,,,F,T,T,,Another Cat Name,,,,,,Another Cat Name,,,1,2,,,,1547,,2017-05-19 01:02:54,731,731,,Another Cat Name,,,Another Cat Name,,,,,,,,,,,,,,1,F,F,F,T,T,,,,,,2017-05-19 19:34:44,F,F,1547,,0,,2017-06-30 0:00:00,,,2017-05-01 00:00:00,,1,0
-1499,FALSE,2022-05-27 13:12:02,,,F,T,T,,Should this be a dog name,,,,,,Should this be a dog name,,,1,2,,,,465,,2017-05-13 22:03:58,1042,1042,,Should this be a dog name,,,Should this be a dog name,,,,,,,,,,,,,,1,F,F,F,T,T,,,,,,2017-05-14 21:05:43,F,F,465,,0,,,,,2017-05-13 00:00:00,,1,0
-1502,FALSE,2022-05-27 13:12:02,,,F,T,T,,Nah only cats here,,,,,,Nah only cats here,,,1,2,,,,1491,,2017-05-21 14:06:47,944,944,,Nah only cats here,,2,Nah only cats here,,,,,,,,,,,,,,1,F,F,F,T,T,,,,,,2017-05-03 22:14:14,F,F,1491,,0,,,,,2017-03-01 00:00:00,,1,0
-1504,FALSE,2022-05-27 13:12:02,,,F,T,T,,You still reading these?,,,,,,You still reading these?,,,1,2,,,,1491,,2017-05-22 01:52:36,498,498,,You still reading these?,,2,You still reading these?,,,,,,,,,,,,,,1,F,F,F,T,T,,,,,,2017-05-03 22:07:58,F,F,1491,,0,,,,,2017-05-21 00:00:00,,1,0
+1548,FALSE,2022-05-27 13:12:02,,,F,T,T,,Another Cat Name,,,,,,Another Cat Name,,,1,2,,,,1547,,2017-05-19 1:02:54,731,731,,Another Cat Name,,,Another Cat Name,,,,,,,,,,,,,,1,F,F,F,T,T,,,,,,2017-05-19 19:34:44,F,F,1547,,0,,2017-06-30 0:00:00,,,2017-05-01 0:00:00,,1,0
+1499,FALSE,2022-05-27 13:12:02,,,F,T,T,,Should this be a dog name,,,,,,Should this be a dog name,,,1,2,,,,465,,2017-05-13 22:03:58,1042,1042,,Should this be a dog name,,,Should this be a dog name,,,,,,,,,,,,,,1,F,F,F,T,T,,,,,,2017-05-14 21:05:43,F,F,465,,0,,,,,2017-05-13 0:00:00,,1,0
+1502,FALSE,2022-05-27 13:12:02,,,F,T,T,,Nah only cats here,,,,,,Nah only cats here,,,1,2,,,,1491,,2017-05-21 14:06:47,944,944,,Nah only cats here,,2,Nah only cats here,,,,,,,,,,,,,,1,F,F,F,T,T,,,,,,2017-05-03 22:14:14,F,F,1491,,0,,,,,2017-03-01 0:00:00,,1,0
+1504,FALSE,2022-05-27 13:12:02,,,F,T,T,,You still reading these?,,,,,,You still reading these?,,,1,2,,,,1491,,2017-05-22 1:52:36,498,498,,You still reading these?,,2,You still reading these?,,,,,,,,,,,,,,1,F,F,F,T,T,,,,,,2017-05-03 22:07:58,F,F,1491,,0,,,,,2017-05-21 0:00:00,,1,0

models/netsuite2.yml

@@ -59,6 +59,7 @@ models:
  - transaction_id
  - accounting_period_id
  - account_name
+ - account_id
  columns:
  - name: transaction_id
  description: Netsuite internal transaction ID.

models/netsuite2/intermediate/base/int_netsuite2__consolidated_exchange_rates.sql

@@ -1,27 +0,0 @@
-{{ config(enabled=(var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2') and var('netsuite2__multibook_accounting_enabled', true) and var('netsuite2__using_exchange_rate', true))) }}
-
-with consolidated_exchange_rates as (
-
- select *
- from {{ var('netsuite2_consolidated_exchange_rates') }}
-),
-
-accounting_book_subsidiaries as (
-
- select *
- from {{ var('netsuite2_accounting_book_subsidiaries') }}
-),
-
-joined as (
-
- select 
- consolidated_exchange_rates.*,
- accounting_book_subsidiaries.accounting_book_id
-
- from consolidated_exchange_rates
- left join accounting_book_subsidiaries
- on consolidated_exchange_rates.to_subsidiary_id = accounting_book_subsidiaries.subsidiary_id
-)
-
-select *
-from joined

models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql

@@ -9,7 +9,7 @@ with accounts as (
 accounting_books as (
  select * 
  from {{ var('netsuite2_accounting_books') }}
-), 
+),
 {% endif %}
 
 subsidiaries as (
@@ -18,8 +18,8 @@ subsidiaries as (
 ),
 
 consolidated_exchange_rates as (
- select * 
- from {{ ref('int_netsuite2__consolidated_exchange_rates') if var('netsuite2__multibook_accounting_enabled', true) else var('netsuite2_consolidated_exchange_rates') }}
+ select *
+ from {{ var('netsuite2_consolidated_exchange_rates') }}
 ),
 
 period_exchange_rate_map as ( -- exchange rates used, by accounting period, to convert to parent subsidiary
@@ -55,5 +55,5 @@ accountxperiod_exchange_rate_map as ( -- account table with exchange rate detail
  cross join period_exchange_rate_map
 )
 
-select * 
+select *
 from accountxperiod_exchange_rate_map

packages.yml

@@ -1,3 +1,3 @@
 packages:
 - package: fivetran/netsuite_source
- version: [">=0.6.0", "<0.7.0"]
+ version: [">=0.7.0", "<0.8.0"]