From f2697dd189fa843c8504667070a7a83780363cf5 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 10 Feb 2023 14:16:23 -0800 Subject: [PATCH 01/21] using_quoted_identifiers added back --- CHANGELOG.md | 5 +++++ dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- macros/sfdc_formula_view.sql | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e23faea..f553123 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# dbt_salesforce_formula_utils v0.8.1 + +## Bug Fixes +- In v0.8.0, the `using_quoted_identifiers` argument in the `sfdc_formula_view()` macro was erroneously removed, making its default value of `False` immutable. See the [v0.7.2 notes](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/CHANGELOG.md#dbt_salesforce_formula_utils-v072) for more about this variable ([PR #65](https://github.com/fivetran/dbt_salesforce_formula_utils/pull/65)). + # dbt_salesforce_formula_utils v0.8.0 ## 🚨 Breaking Changes 🚨: diff --git a/dbt_project.yml b/dbt_project.yml index c0b5fa7..9d94933 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,4 +1,4 @@ name: 'salesforce_formula_utils' -version: '0.8.0' +version: '0.8.1' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 8202f76..846addd 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'salesforce_formula_integration_tests' -version: '0.8.0' +version: '0.8.1' profile: 'integration_tests' config-version: 2 vars: diff --git a/macros/sfdc_formula_view.sql b/macros/sfdc_formula_view.sql index 8645ff0..a3ae419 100644 --- a/macros/sfdc_formula_view.sql +++ b/macros/sfdc_formula_view.sql @@ -1,4 +1,4 @@ -{%- macro sfdc_formula_view(source_table, source_name='salesforce', reserved_table_name=source_table, fields_to_include=none, full_statement_version=true, materialization='view') -%} +{%- macro sfdc_formula_view(source_table, source_name='salesforce', reserved_table_name=source_table, fields_to_include=none, full_statement_version=true, materialization='view', using_quoted_identifiers=False) -%} -- Best practice for this model is to be materialized as view. That is why we have set that here. {{ From f5af8ecd37c26b6fe950cf62d55be7848b427ddf Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 10 Feb 2023 16:16:32 -0800 Subject: [PATCH 02/21] fixes --- CHANGELOG.md | 4 +++- macros/sfdc_formula_view.sql | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f553123..457c4a3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,9 @@ # dbt_salesforce_formula_utils v0.8.1 ## Bug Fixes -- In v0.8.0, the `using_quoted_identifiers` argument in the `sfdc_formula_view()` macro was erroneously removed, making its default value of `False` immutable. See the [v0.7.2 notes](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/CHANGELOG.md#dbt_salesforce_formula_utils-v072) for more about this variable ([PR #65](https://github.com/fivetran/dbt_salesforce_formula_utils/pull/65)). +([PR #77](https://github.com/fivetran/dbt_salesforce_formula_utils/pull/77)) includes the following changes: +- In v0.8.0, the `using_quoted_identifiers` argument in the `sfdc_formula_view()` macro was erroneously removed, making its default value of `False` immutable. It has been reintroduced. +- `using_quoted_identifiers` now uses warehouse-specific quoting syntax. # dbt_salesforce_formula_utils v0.8.0 diff --git a/macros/sfdc_formula_view.sql b/macros/sfdc_formula_view.sql index a3ae419..b4ca1ab 100644 --- a/macros/sfdc_formula_view.sql +++ b/macros/sfdc_formula_view.sql @@ -15,7 +15,10 @@ {% if full_statement_version %} {% if using_quoted_identifiers %} -{%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'), column='"model"', where="\"object\" = '" ~ source_table ~ "'") -%} +{{ print("poop\n") }} +{%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'), + column='"model"' if target.type in ('snowflake', 'postgres', 'redshift') else '`model`', + where=("\"object\" = '" if target.type in ('snowflake', 'postgres', 'redshift') else "`object` = '") ~ source_table ~ "'") -%} {% else %} {%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'), column='model', where="object = '" ~ source_table ~ "'") -%} From 0040acbf657c00ef1a61c5ea22feb194b16f4c74 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 10 Feb 2023 16:17:46 -0800 Subject: [PATCH 03/21] remove testing --- macros/sfdc_formula_view.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/macros/sfdc_formula_view.sql b/macros/sfdc_formula_view.sql index b4ca1ab..677fa15 100644 --- a/macros/sfdc_formula_view.sql +++ b/macros/sfdc_formula_view.sql @@ -15,7 +15,6 @@ {% if full_statement_version %} {% if using_quoted_identifiers %} -{{ print("poop\n") }} {%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'), column='"model"' if target.type in ('snowflake', 'postgres', 'redshift') else '`model`', where=("\"object\" = '" if target.type in ('snowflake', 'postgres', 'redshift') else "`object` = '") ~ source_table ~ "'") -%} From 0b2a31184c14460cc176ea184704f044ee0839e2 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 13 Feb 2023 14:40:42 -0800 Subject: [PATCH 04/21] Readme --- CHANGELOG.md | 5 + README.md | 192 +++++++++++++++++++----------- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 4 files changed, 128 insertions(+), 73 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e23faea..8ade0fb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +# dbt_salesforce_formula_utils v0.8.1 + +## Features +- Updated the README for easier use and navigation of the package (add PR). + # dbt_salesforce_formula_utils v0.8.0 ## 🚨 Breaking Changes 🚨: diff --git a/README.md b/README.md index 43c2fe3..0ea04ad 100644 --- a/README.md +++ b/README.md @@ -11,55 +11,83 @@

# Fivetran Salesforce Formula Utils - +# 📣 What does this dbt package do? This package includes macros to be used within a Salesforce dbt project to accurately map Salesforce Formulas to existing tables. -## Installation Instructions -In order to use this macro it is expected that you have set up your own dbt project. If you have not, you can reference the [dbt installation docs](https://docs.getdbt.com/dbt-cli/installation) for the latest installation instructions, and then reference [the dbt package docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. +# 🎯 How do I use the dbt package? +## Step 1: Prerequisites +To use this dbt package, you must have the following: +- At least one Fivetran Salesforce connector syncing data into your destination. +- A **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination. -## Macro Instructions -### Installing the Macro Package -This macro is intended to be used within a salesforce dbt project model. To leverage the macro, you will add the below configuration to your `packages.yml` file (if you do not have a `packages.yml` file you can create one). +## Step 2: Install the package +This macro is intended to be used within a Salesforce dbt project model. To leverage the macro, you will add the below configuration to your `packages.yml` file (if you do not have a `packages.yml` file you can create one). +> TIP: Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. ```yml packages: - package: fivetran/salesforce_formula_utils - version: [">=0.8.0", "<0.9.0"] -``` -## Source Tables Required -In order to use the macros included in this package, you will need to have a properly configured source package with a source named `salesforce`. An example of a properly configured Salesforce source yml can be found in the `src_salesforce.yml` file in [integration_tests](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/integration_tests/models/src_fivetran_formula.yml). This integration_tests folder is just for testing purposes - your source file will need to be in the dbt root models folder. You are welcome to copy/paste the [example](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/integration_tests/models/src_fivetran_formula.yml) source configuration into your `src_salesforce.yml` file and modify for your use case. In particular, you will need the following sources defined in your `src_salesforce.yml` file: + version: [">=0.8.0", "<0.9.0"] # we recommend using ranges to capture non-breaking changes automatically ``` + +## Step 3: Define required source tables +In order to use the macros included in this package, you will need to have a properly configured Salesforce source named `salesforce` in your own dbt project. An example of a properly configured Salesforce source yml can be found in the `src_salesforce.yml` file in [integration_tests](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/integration_tests/models/src_fivetran_formula.yml). This integration_tests folder is just for testing purposes - your source file will need to be in the models folder of your root dbt project. You are welcome to copy/paste the [example](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/integration_tests/models/src_fivetran_formula.yml) source configuration into your `src_salesforce.yml` file and modify for your use case. + +In particular, you will need the following sources defined in your `src_salesforce.yml` file: +```yml version: 2 sources: - - name: salesforce #It would be best to keep this named salesforce - schema: 'salesforce_schema' #Modify this to be where your Salesforce data resides + - name: salesforce # It would be best to keep this named salesforce + schema: 'salesforce_schema' # Modify this to be where your Salesforce data resides + # Add the database where your Salesforce data resides if different from the target database. Eg. 'my_salesforce_database'. By default the target.database is used. tables: - name: fivetran_formula_model description: Used for the recommended Option 1 version of the formula solution. - name: fivetran_formula description: Used for options 2 and 3 of the original individual formula solution. - ## Any other source tables you are creating models for should be defined here as well. They aren't required it is best organizational practice and allows Fivetran to compile data lineage graphs + ## Any other source tables you are creating models for should be defined here as well. They aren't required, but it is best organizational practice and allows Fivetran to compile data lineage graphs ``` -## Model Creation -### (Recommended and default) Option 1: Generate all relevant formula fields at once -If you would like your model to generate all the formula fields at once related to your source table then you will create a new file in your models folder and name it (`your_table_name_here`.sql). You will then add the below snippet into the file. Finally, update the `source_table` argument to be the source table name for which you are generating the model: + +## Step 4: Create models +### (Recommended and default) Option 1: Generate all relevant formula fields using connector-made query +> This option makes use of the `fivetran_formula_model` lookup table, which stores connector-generated SQL queries for each source table. Compared to `fivetran_formula`, which is used in Options 2 & 3, it is typically more complete and supports nested formulas. + +If you would like your model to generate all the formula fields at once related to your source table then you will need to: +1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). +2. Add the below snippet calling the `sfdc_formula_view` macro into the file. Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). ```sql {{ salesforce_formula_utils.sfdc_formula_view( source_table='your_source_table_name_here', full_statement_version=true) }} ``` -### Option 2: Generate all relevant formula fields individually -If you would like your model to generate all the formula fields related to your source table then you may create a new file in your models folder and name it (`your_table_name_here`.sql). You will then add the below snippet into the file. Finally, update the `source_table` argument to be the source table name for which you are generating the model: + +**Output**: All formulas for the chosen source table will be generated in a single row `select` statement. + +### Option 2: Generate all relevant formula fields using package-made query +> This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas and may be incomplete compared to Option 1. + +If you would like your model to generate all the formula fields related to your source table then you will need to: +1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). +2. Add the below snippet calling the `sfdc_formula_view` macro into the file. Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). ```sql {{ salesforce_formula_utils.sfdc_formula_view( source_table='your_source_table_name_here', full_statement_version=false) }} ``` -### Option 3: Generate only specified formula fields -If you would like your model to generate only a specified subset of your formula fields related to your source table then you may create a new file in your models folder and name it (`your_table_name_here`.sql). You will then add the below snippet into the file. Finally, update the `source_table` argument to be the source table name for which you are generating the model and update the `fields_to_include` argument to contain all the fields from your source that you would like to be included in the final output. Be sure that the field(s) you would like to include are enclosed within brackets as an array (ie. `[]`): + +**Output**: All formulas for the chosen source table will be generated in a `select` statement. This output differs from Option 1 only in the format of compiled code. + +### Option 3: Generate only specified formula fields using package-made query +> This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas. + +If you would like your model to generate only a specified subset of your formula fields related to your source table then you will need to: +1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). +2. Add the below snippet calling the `sfdc_formula_view` macro into the file. + a) Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). + b) Update the `fields_to_include` argument to contain all the fields from your source that you would like to be included in the final output. Be sure that the field(s) you would like to include are enclosed within brackets as an array (ie. `[]`) ```sql {{ salesforce_formula_utils.sfdc_formula_view( source_table='your_source_table_name_here', @@ -67,30 +95,37 @@ If you would like your model to generate only a specified subset of your formula full_statement_version=false) }} ``` -### Formula Fields that Reference Other Formula Fields -This macro has been created to allow for two degrees of formula field reference. For example: + +**Output**: Formulas provided in the `fields_to_include` variable will be generated in a `select` statement for the chosen source table. + +### Automate model creation +If you have multiple models you need to create, you can also Leverage the [sfdc_formula_model_automation](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/sfdc_formula_model_automation.sh) script within this project to automatically create models locally via the command line. Below is an example command to copy and edit. + +```bash +source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh "../path/to/directory" "desired_table_1,desired_table_2,desired_table_infinity" +``` + +By default, these models will run Option #1. + +## Step 5: Exclude problematic formula fields +The `sfdc_formula_view` macro has been created to allow for two degrees of formula field reference. For example: - :white_check_mark: : A formula field references standard fields from the base Salesforce table. -- :white_check_mark: : A formula field references another formula field that does not reference other formula fields. -- 🚧 : A formula field references another formula field that references another formula field (etc.). This may be possible for certain situations using Option #1 above. +- :white_check_mark: : A formula field references another formula field that does **not** reference other formula fields. +- 🚧 : A formula field references another formula field that references another formula field (and so on...). This may be possible for certain situations using Option #1 above, but never for Options 2 & 3. -If you have a formula field that would fall under the :x: example, exclude it from all your models by configuring the `sfdc_exclude_formulas` variable within your root `dbt_project.yml` file. Configure this variable as a set of all the fields you would like to exclude from all models. See below for an example: +If you have a formula field that is double-nested or is otherwise not compiling, exclude it from all your models by setting the `sfdc_exclude_formulas` variable within your root `dbt_project.yml` file. Configure this variable as a set of all the fields you would like to exclude from all models. See below for an example: ```yml vars: sfdc_exclude_formulas: ('field_that_references_other_formula','other_triple_ref_field','field_i_just_dont_want') ``` -> **Note**: You should not add this variable to your `dbt_project.yml` file if you do not need to exclude any fields. +> **Note**: Do not add this variable to your `dbt_project.yml` file if you do not need to exclude any fields. +## Step 6: Execute models Once you have created all your desired models and copied/modified the sql snippet into each model you will execute `dbt deps` to install the macro package, then execute `dbt run` to generate the models. Additionally, you can reference the [integration_tests](https://github.com/fivetran/dbt_salesforce_formula_utils/tree/main/integration_tests/models) folder for examples on how to use the macro within your models. -### Model Creation Automation -If you have multiple models you need to create, you can also Leverage the [sfdc_formula_model_automation](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/sfdc_formula_model_automation.sh) script within this project to automatically create models locally via the command line. Below is an example command to copy and edit. +## Step 7: Read macro & script documentation -```bash -source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh "../path/to/directory" "desired_table_1,desired_table_2,desired_table_infinity" -``` - -## Macro Descriptions -### sfdc_formula_view ([source](macros/sfdc_formula_view.sql)) +### Macro: sfdc_formula_view ([source](macros/sfdc_formula_view.sql)) This macro generates the final sql needed to join the Salesforce formula fields to the desired table. **Usage:** @@ -107,7 +142,25 @@ This macro generates the final sql needed to join the Salesforce formula fields > Note: If you populate the `fields_to_include` argument then the package will exclusively look for those fields. If you have designated a field to be excluded within the `sfdc_exclude_formulas` variable, then this will be ignored and the field will be included in the final model. ---- -### sfdc_formula_pivot ([source](macros/sfdc_formula_pivot.sql)) +### Automation bash script: sfdc_formula_model_automation.sh ([source](sfdc_formula_model_automation.sh)) +This bash script is intended to be used in order to automatically create the desired salesforce models via the command line within your dbt project. This bash script will generate a model file within your dbt project that contains the [sfdc_formula_view](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_formula_view.sql) macro for the appropriately defined table(s). In order for this command to work you must be within the root directory of your dbt project. + +**Usage:** +```bash +source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh "../path/to/directory" "desired_table(s)" +``` + +**Example** +Assuming the path to your directory is `"../dbt_salesforce"` and the table(s) you want to generate the model for are `opportunity` and `account`. +```bash +source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh "../dbt_salesforce" "opportunity,account" +``` + +### Under the hood macros +
Expand to view documenation +
+ +#### sfdc_formula_pivot ([source](macros/sfdc_formula_pivot.sql)) This macro pivots the dictionary results generated from the [sfdc_get_formula_column_values](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_get_formula_column_values.sql) macro to populate the formula field and sql for each record within the designated table this macro is used. **Usage:** @@ -120,7 +173,7 @@ This macro pivots the dictionary results generated from the [sfdc_get_formula_co * `added_inclusion_fields` (optional, default is none): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. ---- -### sfdc_formula_refactor ([source](macros/sfdc_formula_refactor.sql)) +#### sfdc_formula_refactor ([source](macros/sfdc_formula_refactor.sql)) This macro checks the dictionary results generated from the [sfdc_get_formula_column_values](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_get_formula_column_values.sql) macro to determine if any formula fields reference other formula fields. If a formula references another generated formula field, then the macro will insert the first formula sql into the second formula. This ensures formulas that reference another formula field will be generated successfully. **Usage:** @@ -132,7 +185,7 @@ This macro checks the dictionary results generated from the [sfdc_get_formula_co * `source_name` (optional, default is `salesforce`): The name of the source defined. * `added_inclusion_fields` (optional, default is none): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. ---- -### sfdc_formula_view_fields ([source](macros/sfdc_formula_view_fields.sql)) +#### sfdc_formula_view_fields ([source](macros/sfdc_formula_view_fields.sql)) This macro checks the dictionary results generated from the [sfdc_get_formula_column_values](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_get_formula_column_values.sql) macro and returns the field name with the index of the view sql to be used in the primary select statement of the [sfdc_formula_view](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_formula_view.sql) macro. **Usage:** @@ -144,7 +197,7 @@ This macro checks the dictionary results generated from the [sfdc_get_formula_co * `source_name` (optional): The name of the source defined. Default is `salesforce`. * `inclusion_fields` (optional, default is none): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. ---- -### sfdc_formula_view_sql ([source](macros/sfdc_formula_view_sql.sql)) +#### sfdc_formula_view_sql ([source](macros/sfdc_formula_view_sql.sql)) This macro checks the dictionary results generated from the [sfdc_get_formula_column_values](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_get_formula_column_values.sql) macro and returns the view_sql value results while also replacing the `from` and `join` syntax to be specific to the source defined. Additionally, the where logic will be applied to ensure the view_sql is properly joined to the base table. **Usage:** @@ -157,7 +210,7 @@ This macro checks the dictionary results generated from the [sfdc_get_formula_co * `inclusion_fields` (optional, default is none): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. ---- -### sfdc_get_formula_column_values ([source](macros/sfdc_get_formula_column_values.sql)) +#### sfdc_get_formula_column_values ([source](macros/sfdc_get_formula_column_values.sql)) This macro is designed to look within the users source defined `salesforce_schema` for the `fivetran_formula` table. The macro will then filter to only include records from the `join_to_table` argument, and search for distinct combinations of the `field` and `sql` columns. The distinct combination of columns for the join_table argument are then returned as a dictionary. Further, if there are any formula fields that are a third degree referential formula (eg. a formula field is contains a field that is a formula field, and that formula field also references another formula field) or greater then you can add those fields to the `sfdc_exclude_formulas` variable within your root `dbt_project.yml` file to exclude them from the macro as they will fail in the final view creation. > Note: This macro will not work accurately unless you have a `src.yml` configured appropriately. For reference, look within this packages [integration_tests](https://github.com/fivetran/dbt_salesforce_formula_utils/tree/main/integration_tests/models) folder for an example of a source configuration. @@ -175,7 +228,7 @@ Further, if there are any formula fields that are a third degree referential for * `no_nulls` (optional, default is true): Used by the macro to identify if the `null` fields within the `view_sql` column should be included. ---- -### sfdc_star_exact_ ([source](macros/sfdc_star_exact.sql)) +#### sfdc_star_exact ([source](macros/sfdc_star_exact.sql)) This macro mirrors the [dbt_utils.star()](https://github.com/dbt-labs/dbt-utils#star-source) macro with the minor adjustment to properly work with the return results of the [dbt_utils.get_column_values()](https://github.com/dbt-labs/dbt-utils#get_column_values-source). The major change being how the return result of the dbt_utils.get_column_values() macro returns results with single quotes, while the dbt_utils.star() macro exclusively requires double quotes. **Usage:** ```sql @@ -186,40 +239,37 @@ This macro mirrors the [dbt_utils.star()](https://github.com/dbt-labs/dbt-utils# * `relation_alias` (optional, default is none): Used if you would like to add a relation alias to the fields queried in the select star statement (i.e. `my_table.field_name`). * `except` (optional, default is none): A list of fields you would like to exclude from the select star statement. ---- -## Automation Bash Script -### sfdc_formula_model_automation.sh ([source](sfdc_formula_model_automation.sh)) -This bash script is intended to be used in order to automatically create the desired salesforce models via the command line within your dbt project. This bash script will generate a model file within your dbt project that contains the [sfdc_formula_view](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_formula_view.sql) macro for the appropriately defined table(s). In order for this command to work you must be within the root directory of your dbt project. +
-**Usage:** -```bash -source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh "../path/to/directory" "desired_table(s)" -``` +## (Optional) Step 8: Orchestrate your models with Fivetran Transformations for dbt Core™ +
Expand to view details +
+ +Fivetran offers the ability for you to orchestrate your dbt project through [Fivetran Transformations for dbt Core™](https://fivetran.com/docs/transformations/dbt). Learn how to set up your project for orchestration through Fivetran in our [Transformations for dbt Core setup guides](https://fivetran.com/docs/transformations/dbt#setupguide). +
-**Example** -Assuming the path to your directory is `"../dbt_salesforce"` and the table(s) you want to generate the model for are `opportunity` and `account`. -```bash -source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh "../dbt_salesforce" "opportunity,account" +# 🔍 Does this package have dependencies? +This dbt package is dependent on the following dbt packages. Please be aware that these dependencies are installed by default within this package. For more information on the following packages, refer to the [dbt hub](https://hub.getdbt.com/) site. +> IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts. +```yml +packages: + - package: fivetran/fivetran_utils + version: [">=0.4.0", "<0.5.0"] + + - package: dbt-labs/dbt_utils + version: [">=1.0.0", "<2.0.0"] ``` + +# 🙌 How is this package maintained and can I contribute? ## Package Maintenance -The Fivetran team maintaining this package **only** maintains the latest version of the package. We highly recommend you stay consistent with the [latest version](https://hub.getdbt.com/fivetran/salesforce_formula_utils/latest/) of the package and refer to the [CHANGELOG](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/CHANGELOG.md) and release notes for more information on changes across versions. +The Fivetran team maintaining this package _only_ maintains the latest version of the package. We highly recommend that you stay consistent with the [latest version](https://hub.getdbt.com/fivetran/shopify_source/latest/) of the package and refer to the [CHANGELOG](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/CHANGELOG.md) and release notes for more information on changes across versions. ## Contributions -Don't see a model or specific metric you would have liked to be included? Notice any bugs when installing -and running the package? If so, we highly encourage and welcome contributions to this package! -Please create issues or open PRs against `master`. Check out [this post](https://discourse.getdbt.com/t/contributing-to-a-dbt-package/657) on the best workflow for contributing to a package. - -## Database Support -This package has been tested on BigQuery, Snowflake, Redshift, and Postgres. - -## Resources: -- Provide [feedback](https://www.surveymonkey.com/r/DQ7K7WW) on our existing dbt packages or what you'd like to see next -- Have questions, feedback, or need help? Book a time during our office hours [using Calendly](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) or email us at solutions@fivetran.com -- Find all of Fivetran's pre-built dbt packages in our [dbt hub](https://hub.getdbt.com/fivetran/) -- Learn how to orchestrate your models with [Fivetran Transformations for dbt Core™](https://fivetran.com/docs/transformations/dbt) -- Learn more about Fivetran overall [in our docs](https://fivetran.com/docs) -- Check out [Fivetran's blog](https://fivetran.com/blog) -- Learn more about dbt [in the dbt docs](https://docs.getdbt.com/docs/introduction) -- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers -- Join the [chat](http://slack.getdbt.com/) on Slack for live discussions and support -- Find [dbt events](https://events.getdbt.com) near you -- Check out [the dbt blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices +A small team of analytics engineers at Fivetran develops these dbt packages. However, the packages are made better by community contributions! + +We highly encourage and welcome contributions to this package. Check out [this dbt Discourse article](https://discourse.getdbt.com/t/contributing-to-a-dbt-package/657) to learn how to contribute to a dbt package! + +# 🏪 Are there any resources available? +- If you have questions or want to reach out for help, please refer to the [GitHub Issue](https://github.com/fivetran/dbt_salesforce_formula_utils/issues/new/choose) section to find the right avenue of support for you. +- If you would like to provide feedback to the dbt package team at Fivetran or would like to request a new dbt package, fill out our [Feedback Form](https://www.surveymonkey.com/r/DQ7K7WW). +- Have questions or want to just say hi? Book a time during our office hours [on Calendly](https://calendly.com/fivetran-solutions-team/fivetran-solutions-team-office-hours) or email us at solutions@fivetran.com. \ No newline at end of file diff --git a/dbt_project.yml b/dbt_project.yml index c0b5fa7..9d94933 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,4 +1,4 @@ name: 'salesforce_formula_utils' -version: '0.8.0' +version: '0.8.1' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 8202f76..846addd 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'salesforce_formula_integration_tests' -version: '0.8.0' +version: '0.8.1' profile: 'integration_tests' config-version: 2 vars: From 692b69a219efdf1899646d5ddbea5bbda7ae9d71 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 13 Feb 2023 14:55:44 -0800 Subject: [PATCH 05/21] tweaks --- README.md | 79 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 0ea04ad..ecf6467 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,6 @@ sources: ## Step 4: Create models ### (Recommended and default) Option 1: Generate all relevant formula fields using connector-made query -> This option makes use of the `fivetran_formula_model` lookup table, which stores connector-generated SQL queries for each source table. Compared to `fivetran_formula`, which is used in Options 2 & 3, it is typically more complete and supports nested formulas. If you would like your model to generate all the formula fields at once related to your source table then you will need to: 1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). @@ -65,8 +64,9 @@ If you would like your model to generate all the formula fields at once related **Output**: All formulas for the chosen source table will be generated in a single row `select` statement. +> This option makes use of the `fivetran_formula_model` lookup table, which stores connector-generated SQL queries for each source table. Compared to `fivetran_formula`, which is used in Options 2 & 3, it is typically more complete and supports most double-nested formulas. + ### Option 2: Generate all relevant formula fields using package-made query -> This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas and may be incomplete compared to Option 1. If you would like your model to generate all the formula fields related to your source table then you will need to: 1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). @@ -80,14 +80,15 @@ If you would like your model to generate all the formula fields related to your **Output**: All formulas for the chosen source table will be generated in a `select` statement. This output differs from Option 1 only in the format of compiled code. +> This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas and may be incomplete compared to Option #1. + ### Option 3: Generate only specified formula fields using package-made query -> This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas. If you would like your model to generate only a specified subset of your formula fields related to your source table then you will need to: 1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). -2. Add the below snippet calling the `sfdc_formula_view` macro into the file. - a) Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). - b) Update the `fields_to_include` argument to contain all the fields from your source that you would like to be included in the final output. Be sure that the field(s) you would like to include are enclosed within brackets as an array (ie. `[]`) +2. Add the below snippet calling the `sfdc_formula_view` macro into the file and: + - åUpdate the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). + - Update the `fields_to_include` argument to contain all the fields from your source that you would like to be included in the final output. Be sure that the field(s) you would like to include are enclosed within brackets as an array (ie. `[]`) ```sql {{ salesforce_formula_utils.sfdc_formula_view( source_table='your_source_table_name_here', @@ -98,6 +99,8 @@ If you would like your model to generate only a specified subset of your formula **Output**: Formulas provided in the `fields_to_include` variable will be generated in a `select` statement for the chosen source table. +> This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas. + ### Automate model creation If you have multiple models you need to create, you can also Leverage the [sfdc_formula_model_automation](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/sfdc_formula_model_automation.sh) script within this project to automatically create models locally via the command line. Below is an example command to copy and edit. @@ -123,9 +126,16 @@ vars: ## Step 6: Execute models Once you have created all your desired models and copied/modified the sql snippet into each model you will execute `dbt deps` to install the macro package, then execute `dbt run` to generate the models. Additionally, you can reference the [integration_tests](https://github.com/fivetran/dbt_salesforce_formula_utils/tree/main/integration_tests/models) folder for examples on how to use the macro within your models. -## Step 7: Read macro & script documentation +## (Optional) Step 7: Orchestrate your models with Fivetran Transformations for dbt Core™ +
Expand to view details +
+ +Fivetran offers the ability for you to orchestrate your dbt project through [Fivetran Transformations for dbt Core™](https://fivetran.com/docs/transformations/dbt). Learn how to set up your project for orchestration through Fivetran in our [Transformations for dbt Core setup guides](https://fivetran.com/docs/transformations/dbt#setupguide). +
+ +# :book: Macro & script documentation -### Macro: sfdc_formula_view ([source](macros/sfdc_formula_view.sql)) +## Macro: sfdc_formula_view ([source](macros/sfdc_formula_view.sql)) This macro generates the final sql needed to join the Salesforce formula fields to the desired table. **Usage:** @@ -134,15 +144,15 @@ This macro generates the final sql needed to join the Salesforce formula fields ``` **Args:** * `source_table` (required): The table with which you are joining the formula fields. -* `source_name` (optional, default 'salesforce'): The dbt source containing the table you want to join with formula fields. Must also contain the `fivetran_formula` table. -* `fields_to_include` (optional, default is none): If a users wishes to only run the formula fields macro for designated fields then they may be applied within this variable. This variable will ensure the model only generates the sql for the designated fields. -* `full_statement_version` (optional, default is true): Allows a user to leverage the `fivetran_formula_table` version of the macro which will generate the formula fields via the complete sql statement, rather than individual formulas being generated within the macro. -* `using_quoted_identifiers` (optional, default is false): For warehouses with case sensitivity enabled this argument **must** be set to `true` in order for the underlying macros within this project to properly compile and execute successfully. -* `materialization` (optional, default is `view`): By default the model will be materialized as a view. If you would like to materialize as a table, you can adjust using this argument. +* `source_name` (optional, default = `'salesforce'`): The dbt source containing the table you want to join with formula fields. Must also contain the `fivetran_formula` table. +* `fields_to_include` (optional, default = `none`): If a users wishes to only run the formula fields macro for designated fields then they may be applied within this variable. This variable will ensure the model only generates the sql for the designated fields. +* `full_statement_version` (optional, default = `true`): Allows a user to leverage the `fivetran_formula_table` version of the macro which will generate the formula fields via the complete sql statement, rather than individual formulas being generated within the macro. +* `using_quoted_identifiers` (optional, default = `false`): For warehouses with case sensitivity enabled this argument **must** be set to `true` in order for the underlying macros within this project to properly compile and execute successfully. +* `materialization` (optional, default = `view`): By default the model will be materialized as a view. If you would like to materialize as a table, you can adjust using this argument. > Note: If you populate the `fields_to_include` argument then the package will exclusively look for those fields. If you have designated a field to be excluded within the `sfdc_exclude_formulas` variable, then this will be ignored and the field will be included in the final model. ---- -### Automation bash script: sfdc_formula_model_automation.sh ([source](sfdc_formula_model_automation.sh)) +## Automation bash script: sfdc_formula_model_automation.sh ([source](sfdc_formula_model_automation.sh)) This bash script is intended to be used in order to automatically create the desired salesforce models via the command line within your dbt project. This bash script will generate a model file within your dbt project that contains the [sfdc_formula_view](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_formula_view.sql) macro for the appropriately defined table(s). In order for this command to work you must be within the root directory of your dbt project. **Usage:** @@ -156,11 +166,11 @@ Assuming the path to your directory is `"../dbt_salesforce"` and the table(s) yo source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh "../dbt_salesforce" "opportunity,account" ``` -### Under the hood macros +## Under the hood macros
Expand to view documenation
-#### sfdc_formula_pivot ([source](macros/sfdc_formula_pivot.sql)) +### sfdc_formula_pivot ([source](macros/sfdc_formula_pivot.sql)) This macro pivots the dictionary results generated from the [sfdc_get_formula_column_values](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_get_formula_column_values.sql) macro to populate the formula field and sql for each record within the designated table this macro is used. **Usage:** @@ -169,11 +179,11 @@ This macro pivots the dictionary results generated from the [sfdc_get_formula_co ``` **Args:** * `join_to_table` (required): The table with which you are joining the formula fields. -* `source_name` (optional, default 'salesforce'): The dbt source containing the table you want to join with formula fields. Must also contain the `fivetran_formula` table. -* `added_inclusion_fields` (optional, default is none): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. +* `source_name` (optional, default = `'salesforce'`): The dbt source containing the table you want to join with formula fields. Must also contain the `fivetran_formula` table. +* `added_inclusion_fields` (optional, default = `none`): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. ---- -#### sfdc_formula_refactor ([source](macros/sfdc_formula_refactor.sql)) +### sfdc_formula_refactor ([source](macros/sfdc_formula_refactor.sql)) This macro checks the dictionary results generated from the [sfdc_get_formula_column_values](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_get_formula_column_values.sql) macro to determine if any formula fields reference other formula fields. If a formula references another generated formula field, then the macro will insert the first formula sql into the second formula. This ensures formulas that reference another formula field will be generated successfully. **Usage:** @@ -182,10 +192,10 @@ This macro checks the dictionary results generated from the [sfdc_get_formula_co ``` **Args:** * `join_to_table` (required): The table with which you are joining the formula fields. -* `source_name` (optional, default is `salesforce`): The name of the source defined. -* `added_inclusion_fields` (optional, default is none): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. +* `source_name` (optional, default = `salesforce`): The name of the source defined. +* `added_inclusion_fields` (optional, default = `none`): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. ---- -#### sfdc_formula_view_fields ([source](macros/sfdc_formula_view_fields.sql)) +### sfdc_formula_view_fields ([source](macros/sfdc_formula_view_fields.sql)) This macro checks the dictionary results generated from the [sfdc_get_formula_column_values](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_get_formula_column_values.sql) macro and returns the field name with the index of the view sql to be used in the primary select statement of the [sfdc_formula_view](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_formula_view.sql) macro. **Usage:** @@ -195,9 +205,9 @@ This macro checks the dictionary results generated from the [sfdc_get_formula_co **Args:** * `join_to_table` (required): The table with which you are joining the formula fields. * `source_name` (optional): The name of the source defined. Default is `salesforce`. -* `inclusion_fields` (optional, default is none): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. +* `inclusion_fields` (optional, default = `none`): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. ---- -#### sfdc_formula_view_sql ([source](macros/sfdc_formula_view_sql.sql)) +### sfdc_formula_view_sql ([source](macros/sfdc_formula_view_sql.sql)) This macro checks the dictionary results generated from the [sfdc_get_formula_column_values](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_get_formula_column_values.sql) macro and returns the view_sql value results while also replacing the `from` and `join` syntax to be specific to the source defined. Additionally, the where logic will be applied to ensure the view_sql is properly joined to the base table. **Usage:** @@ -207,10 +217,10 @@ This macro checks the dictionary results generated from the [sfdc_get_formula_co **Args:** * `join_to_table` (required): The table with which you are joining the formula fields. * `source_name` (optional): The name of the source defined. Default is `salesforce`. -* `inclusion_fields` (optional, default is none): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. +* `inclusion_fields` (optional, default = `none`): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. ---- -#### sfdc_get_formula_column_values ([source](macros/sfdc_get_formula_column_values.sql)) +### sfdc_get_formula_column_values ([source](macros/sfdc_get_formula_column_values.sql)) This macro is designed to look within the users source defined `salesforce_schema` for the `fivetran_formula` table. The macro will then filter to only include records from the `join_to_table` argument, and search for distinct combinations of the `field` and `sql` columns. The distinct combination of columns for the join_table argument are then returned as a dictionary. Further, if there are any formula fields that are a third degree referential formula (eg. a formula field is contains a field that is a formula field, and that formula field also references another formula field) or greater then you can add those fields to the `sfdc_exclude_formulas` variable within your root `dbt_project.yml` file to exclude them from the macro as they will fail in the final view creation. > Note: This macro will not work accurately unless you have a `src.yml` configured appropriately. For reference, look within this packages [integration_tests](https://github.com/fivetran/dbt_salesforce_formula_utils/tree/main/integration_tests/models) folder for an example of a source configuration. @@ -224,11 +234,11 @@ Further, if there are any formula fields that are a third degree referential for * `key` (required): The key column within `fivetran_formula` you are querying. This argument is typically `field`. * `value` (required): The value column within `fivetran_formula` you are querying. This argument is typically `sql`. * `join_to_table` (required): The table with which you are joining the formula fields. -* `added_inclusion_fields` (optional, default is none): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. -* `no_nulls` (optional, default is true): Used by the macro to identify if the `null` fields within the `view_sql` column should be included. +* `added_inclusion_fields` (optional, default = `none`): The list of fields you want to be included in the macro. If no fields are selected then all fields will be included. +* `no_nulls` (optional, default = `true`): Used by the macro to identify if the `null` fields within the `view_sql` column should be included. ---- -#### sfdc_star_exact ([source](macros/sfdc_star_exact.sql)) +### sfdc_star_exact ([source](macros/sfdc_star_exact.sql)) This macro mirrors the [dbt_utils.star()](https://github.com/dbt-labs/dbt-utils#star-source) macro with the minor adjustment to properly work with the return results of the [dbt_utils.get_column_values()](https://github.com/dbt-labs/dbt-utils#get_column_values-source). The major change being how the return result of the dbt_utils.get_column_values() macro returns results with single quotes, while the dbt_utils.star() macro exclusively requires double quotes. **Usage:** ```sql @@ -236,18 +246,11 @@ This macro mirrors the [dbt_utils.star()](https://github.com/dbt-labs/dbt-utils# ``` **Args:** * `from` (required): The table which you want to select all fields. -* `relation_alias` (optional, default is none): Used if you would like to add a relation alias to the fields queried in the select star statement (i.e. `my_table.field_name`). -* `except` (optional, default is none): A list of fields you would like to exclude from the select star statement. +* `relation_alias` (optional, default = `none`): Used if you would like to add a relation alias to the fields queried in the select star statement (i.e. `my_table.field_name`). +* `except` (optional, default = `none`): A list of fields you would like to exclude from the select star statement. ----
-## (Optional) Step 8: Orchestrate your models with Fivetran Transformations for dbt Core™ -
Expand to view details -
- -Fivetran offers the ability for you to orchestrate your dbt project through [Fivetran Transformations for dbt Core™](https://fivetran.com/docs/transformations/dbt). Learn how to set up your project for orchestration through Fivetran in our [Transformations for dbt Core setup guides](https://fivetran.com/docs/transformations/dbt#setupguide). -
- # 🔍 Does this package have dependencies? This dbt package is dependent on the following dbt packages. Please be aware that these dependencies are installed by default within this package. For more information on the following packages, refer to the [dbt hub](https://hub.getdbt.com/) site. > IMPORTANT: If you have any of these dependent packages in your own `packages.yml` file, we highly recommend that you remove them from your root `packages.yml` to avoid package version conflicts. From 828a1f8f1e4c972f8d704f1aee39842f8751cb69 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 13 Feb 2023 15:03:15 -0800 Subject: [PATCH 06/21] add pr link --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ade0fb..31399ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_salesforce_formula_utils v0.8.1 ## Features -- Updated the README for easier use and navigation of the package (add PR). +- Updated the README for easier use and navigation of the package ([PR #78](https://github.com/fivetran/dbt_salesforce_formula_utils/pull/78)). # dbt_salesforce_formula_utils v0.8.0 From ab5e9f92184378935c35bb3173a14f1902ed1563 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 13 Feb 2023 15:17:16 -0800 Subject: [PATCH 07/21] updates --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ecf6467..7aa3e6c 100644 --- a/README.md +++ b/README.md @@ -87,8 +87,8 @@ If you would like your model to generate all the formula fields related to your If you would like your model to generate only a specified subset of your formula fields related to your source table then you will need to: 1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). 2. Add the below snippet calling the `sfdc_formula_view` macro into the file and: - - åUpdate the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). - - Update the `fields_to_include` argument to contain all the fields from your source that you would like to be included in the final output. Be sure that the field(s) you would like to include are enclosed within brackets as an array (ie. `[]`) + - Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). + - Update the `fields_to_include` argument to contain all the fields from your source that you would like to be included in the final output. Be sure that the field(s) you would like to include are enclosed within brackets as an array (ie. `[]`) ```sql {{ salesforce_formula_utils.sfdc_formula_view( source_table='your_source_table_name_here', @@ -135,7 +135,7 @@ Fivetran offers the ability for you to orchestrate your dbt project through [Fiv # :book: Macro & script documentation -## Macro: sfdc_formula_view ([source](macros/sfdc_formula_view.sql)) +## sfdc_formula_view ([source](macros/sfdc_formula_view.sql)) This macro generates the final sql needed to join the Salesforce formula fields to the desired table. **Usage:** @@ -152,7 +152,7 @@ This macro generates the final sql needed to join the Salesforce formula fields > Note: If you populate the `fields_to_include` argument then the package will exclusively look for those fields. If you have designated a field to be excluded within the `sfdc_exclude_formulas` variable, then this will be ignored and the field will be included in the final model. ---- -## Automation bash script: sfdc_formula_model_automation.sh ([source](sfdc_formula_model_automation.sh)) +## sfdc_formula_model_automation.sh ([source](sfdc_formula_model_automation.sh)) This bash script is intended to be used in order to automatically create the desired salesforce models via the command line within your dbt project. This bash script will generate a model file within your dbt project that contains the [sfdc_formula_view](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_formula_view.sql) macro for the appropriately defined table(s). In order for this command to work you must be within the root directory of your dbt project. **Usage:** From 3f0988eea4b0c93b6457887ee0fb9c6855c06a50 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 14 Feb 2023 09:56:23 -0800 Subject: [PATCH 08/21] Update README.md Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7aa3e6c..f766aeb 100644 --- a/README.md +++ b/README.md @@ -53,7 +53,7 @@ sources: ### (Recommended and default) Option 1: Generate all relevant formula fields using connector-made query If you would like your model to generate all the formula fields at once related to your source table then you will need to: -1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). +1. Create a new file in your models folder and name it `your_table_name_here.sql` (e.g. `customer.sql`; this is not necessary but recommended as best practice). 2. Add the below snippet calling the `sfdc_formula_view` macro into the file. Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). ```sql {{ salesforce_formula_utils.sfdc_formula_view( From 6a4040ca11a4337cd54af55495dc80dd5542a86d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 14 Feb 2023 09:56:44 -0800 Subject: [PATCH 09/21] Update README.md Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f766aeb..cfa1b6b 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,7 @@ If you would like your model to generate all the formula fields at once related ### Option 2: Generate all relevant formula fields using package-made query If you would like your model to generate all the formula fields related to your source table then you will need to: -1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). +1. Create a new file in your models folder and name it `your_table_name_here.sql` (e.g. `customer.sql`; this is not necessary but recommended as best practice). 2. Add the below snippet calling the `sfdc_formula_view` macro into the file. Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). ```sql {{ salesforce_formula_utils.sfdc_formula_view( From ca073acc4abd1d80d5ab9ebf75b66c8e9d50461d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 14 Feb 2023 09:56:58 -0800 Subject: [PATCH 10/21] Update README.md Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cfa1b6b..104104e 100644 --- a/README.md +++ b/README.md @@ -85,7 +85,7 @@ If you would like your model to generate all the formula fields related to your ### Option 3: Generate only specified formula fields using package-made query If you would like your model to generate only a specified subset of your formula fields related to your source table then you will need to: -1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). +1. Create a new file in your models folder and name it `your_table_name_here.sql` (e.g. `customer.sql`; this is not necessary but recommended as best practice). 2. Add the below snippet calling the `sfdc_formula_view` macro into the file and: - Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). - Update the `fields_to_include` argument to contain all the fields from your source that you would like to be included in the final output. Be sure that the field(s) you would like to include are enclosed within brackets as an array (ie. `[]`) From a7921d64678c4a9433df842ff1fc823ae423b295 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 14 Feb 2023 09:57:57 -0800 Subject: [PATCH 11/21] Update README.md Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 104104e..7db6e68 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ If you would like your model to generate all the formula fields related to your }} ``` -**Output**: All formulas for the chosen source table will be generated in a `select` statement. This output differs from Option 1 only in the format of compiled code. +**Output**: All formulas for the chosen source table will be included in the resulting `select` statement. > This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas and may be incomplete compared to Option #1. From 2630d7f52e234791e5d76d728a77e0f6d9c8d968 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 14 Feb 2023 10:10:27 -0800 Subject: [PATCH 12/21] Update README.md Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7db6e68..ed3d20a 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ If you would like your model to generate all the formula fields at once related }} ``` -**Output**: All formulas for the chosen source table will be generated in a single row `select` statement. +**Output**: All formulas for the chosen source table will be included in the resulting `select` statement. > This option makes use of the `fivetran_formula_model` lookup table, which stores connector-generated SQL queries for each source table. Compared to `fivetran_formula`, which is used in Options 2 & 3, it is typically more complete and supports most double-nested formulas. From 5f4b73fd117cc4b17eab3f179a13dbcf177085bd Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 14 Feb 2023 10:17:49 -0800 Subject: [PATCH 13/21] Update README.md Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ed3d20a..dbcf35b 100644 --- a/README.md +++ b/README.md @@ -97,7 +97,7 @@ If you would like your model to generate only a specified subset of your formula }} ``` -**Output**: Formulas provided in the `fields_to_include` variable will be generated in a `select` statement for the chosen source table. +**Output**: Only formulas provided in the `fields_to_include` variable will be included in the resulting `select` statement for the chosen source table. > This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas. From 6518732e553e8ab7b0a129f0dac4457377489c90 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 14 Feb 2023 10:23:19 -0800 Subject: [PATCH 14/21] Update README.md Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index dbcf35b..9392f0f 100644 --- a/README.md +++ b/README.md @@ -111,7 +111,7 @@ source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh ".. By default, these models will run Option #1. ## Step 5: Exclude problematic formula fields -The `sfdc_formula_view` macro has been created to allow for two degrees of formula field reference. For example: +The `sfdc_formula_view` macro has been created to support double-nested formula field references for most cases using Option #1, however, never if using Options #2 or #3. For example: - :white_check_mark: : A formula field references standard fields from the base Salesforce table. - :white_check_mark: : A formula field references another formula field that does **not** reference other formula fields. - 🚧 : A formula field references another formula field that references another formula field (and so on...). This may be possible for certain situations using Option #1 above, but never for Options 2 & 3. From 5bdf7726b0f090fb2379b96059cef3d53c036d03 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 14 Feb 2023 10:24:55 -0800 Subject: [PATCH 15/21] Update README.md Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 9392f0f..b6f3136 100644 --- a/README.md +++ b/README.md @@ -114,7 +114,8 @@ By default, these models will run Option #1. The `sfdc_formula_view` macro has been created to support double-nested formula field references for most cases using Option #1, however, never if using Options #2 or #3. For example: - :white_check_mark: : A formula field references standard fields from the base Salesforce table. - :white_check_mark: : A formula field references another formula field that does **not** reference other formula fields. -- 🚧 : A formula field references another formula field that references another formula field (and so on...). This may be possible for certain situations using Option #1 above, but never for Options 2 & 3. +The `sfdc_formula_view` macro may be able to support some cases of n-nested formula field references if using Option #1 above, however, never if using Options #2 or #3. For example: +- 🚧 : A formula field references another formula field that references another formula field (and so on...). If you have a formula field that is double-nested or is otherwise not compiling, exclude it from all your models by setting the `sfdc_exclude_formulas` variable within your root `dbt_project.yml` file. Configure this variable as a set of all the fields you would like to exclude from all models. See below for an example: ```yml From 372c5f12d62bb605e0c8bfe6f02c0d9e89c70b54 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 14 Feb 2023 10:46:56 -0800 Subject: [PATCH 16/21] Sheri feedback --- README.md | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 7aa3e6c..db68025 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,9 @@ # Fivetran Salesforce Formula Utils # 📣 What does this dbt package do? -This package includes macros to be used within a Salesforce dbt project to accurately map Salesforce Formulas to existing tables. +This package includes macros and scipts to be used within a dbt project to accurately map Salesforce Formulas to existing tables. It is designed to work with data from [Fivetran's Salesforce connector](https://fivetran.com/docs/applications/salesforce) in the format described by [this ERD](https://fivetran.com/docs/applications/salesforce#schema). + +> Note: this package is distinct from the [Salesforce dbt package](https://github.com/fivetran/dbt_salesforce), which _transforms_ Salesforce data and outputs analytics-ready end models. # 🎯 How do I use the dbt package? ## Step 1: Prerequisites @@ -21,7 +23,7 @@ To use this dbt package, you must have the following: - A **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination. ## Step 2: Install the package -This macro is intended to be used within a Salesforce dbt project model. To leverage the macro, you will add the below configuration to your `packages.yml` file (if you do not have a `packages.yml` file you can create one). +The [`sfdc_formula_view`](https://github.com/fivetran/dbt_salesforce_formula_utils#sfdc_formula_view-source) macro is intended to be leveraged within a dbt project that uses the source tables from Fivetran's Salesforce connector. To leverage the macro, you will add the below configuration to your `packages.yml` file (if you do not have a `packages.yml` file, create on in your root dbt project). > TIP: Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. ```yml packages: @@ -38,7 +40,7 @@ version: 2 sources: - name: salesforce # It would be best to keep this named salesforce - schema: 'salesforce_schema' # Modify this to be where your Salesforce data resides + schema: "{{ var('salesforce_schema', 'salesforce') }}" # Configure the salesforce_schema var from your dbt_project.yml (alternatively you can hard-code the schema here if only using one Salesforce connector) # Add the database where your Salesforce data resides if different from the target database. Eg. 'my_salesforce_database'. By default the target.database is used. tables: - name: fivetran_formula_model @@ -54,7 +56,7 @@ sources: If you would like your model to generate all the formula fields at once related to your source table then you will need to: 1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). -2. Add the below snippet calling the `sfdc_formula_view` macro into the file. Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). +2. Add the below snippet calling the [`sfdc_formula_view`](https://github.com/fivetran/dbt_salesforce_formula_utils#sfdc_formula_view-source) macro into the file. Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). Though `full_statement_version` is explicitly included here, it is `true` by default. ```sql {{ salesforce_formula_utils.sfdc_formula_view( source_table='your_source_table_name_here', @@ -70,11 +72,11 @@ If you would like your model to generate all the formula fields at once related If you would like your model to generate all the formula fields related to your source table then you will need to: 1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). -2. Add the below snippet calling the `sfdc_formula_view` macro into the file. Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). +2. Add the below snippet calling the [`sfdc_formula_view`](https://github.com/fivetran/dbt_salesforce_formula_utils#sfdc_formula_view-source) macro into the file. Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). ```sql {{ salesforce_formula_utils.sfdc_formula_view( source_table='your_source_table_name_here', - full_statement_version=false) + full_statement_version=false) }} ``` @@ -86,7 +88,7 @@ If you would like your model to generate all the formula fields related to your If you would like your model to generate only a specified subset of your formula fields related to your source table then you will need to: 1. Create a new file in your models folder and name it `your_table_name_here`.sql (e.g. `customer.sql`; this is not necessary but recommended as best practice). -2. Add the below snippet calling the `sfdc_formula_view` macro into the file and: +2. Add the below snippet calling the [`sfdc_formula_view`](https://github.com/fivetran/dbt_salesforce_formula_utils#sfdc_formula_view-source) macro into the file and: - Update the `source_table` argument to be the source table name for which you are generating the model (e.g. `customer`). - Update the `fields_to_include` argument to contain all the fields from your source that you would like to be included in the final output. Be sure that the field(s) you would like to include are enclosed within brackets as an array (ie. `[]`) ```sql @@ -102,13 +104,15 @@ If you would like your model to generate only a specified subset of your formula > This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas. ### Automate model creation -If you have multiple models you need to create, you can also Leverage the [sfdc_formula_model_automation](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/sfdc_formula_model_automation.sh) script within this project to automatically create models locally via the command line. Below is an example command to copy and edit. +If you have multiple models you need to create, you can also leverage the [sfdc_formula_model_automation](https://github.com/fivetran/dbt_salesforce_formula_utils#sfdc_formula_model_automationsh-source) script within this project to automatically create models locally via the command line. Below is an example command to copy and edit. ```bash source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh "../path/to/directory" "desired_table_1,desired_table_2,desired_table_infinity" ``` -By default, these models will run Option #1. +**Output**: Model files for each table, populated with `{{ salesforce_formula_utils.sfdc_formula_view(source_table='table_name') }}`. By default, these models will run Option #1. To use Options #2 or #3, you will need to add `full_statement_version=false`. For Option #3, you will need to add the `fields_to_include` argument(s). + +> Note: In order for this command to work, you must currently be within the root directory of your dbt project. ## Step 5: Exclude problematic formula fields The `sfdc_formula_view` macro has been created to allow for two degrees of formula field reference. For example: @@ -144,16 +148,16 @@ This macro generates the final sql needed to join the Salesforce formula fields ``` **Args:** * `source_table` (required): The table with which you are joining the formula fields. -* `source_name` (optional, default = `'salesforce'`): The dbt source containing the table you want to join with formula fields. Must also contain the `fivetran_formula` table. -* `fields_to_include` (optional, default = `none`): If a users wishes to only run the formula fields macro for designated fields then they may be applied within this variable. This variable will ensure the model only generates the sql for the designated fields. -* `full_statement_version` (optional, default = `true`): Allows a user to leverage the `fivetran_formula_table` version of the macro which will generate the formula fields via the complete sql statement, rather than individual formulas being generated within the macro. +* `source_name` (optional, default = `'salesforce'`): The dbt source containing the table you want to join with formula fields (as defined [here](https://github.com/fivetran/dbt_salesforce_formula_utils/tree/main#step-3-define-required-source-tables)). Must contain the `fivetran_formula` and `fivetran_formula_model` tables. +* `fields_to_include` (optional, default = `none`): If a users wishes to only run the formula fields macro for designated fields then they may be applied within this variable. This variable will ensure the model only generates the sql for the designated fields. `full_statement_version` **must** be `false` for this variable to work. +* `full_statement_version` (optional, default = `true`): Allows a user to leverage the `fivetran_formula_model` version of the macro which will generate the formula fields via the complete sql statement, rather than individual formulas being generated within the macro. * `using_quoted_identifiers` (optional, default = `false`): For warehouses with case sensitivity enabled this argument **must** be set to `true` in order for the underlying macros within this project to properly compile and execute successfully. * `materialization` (optional, default = `view`): By default the model will be materialized as a view. If you would like to materialize as a table, you can adjust using this argument. > Note: If you populate the `fields_to_include` argument then the package will exclusively look for those fields. If you have designated a field to be excluded within the `sfdc_exclude_formulas` variable, then this will be ignored and the field will be included in the final model. ---- ## sfdc_formula_model_automation.sh ([source](sfdc_formula_model_automation.sh)) -This bash script is intended to be used in order to automatically create the desired salesforce models via the command line within your dbt project. This bash script will generate a model file within your dbt project that contains the [sfdc_formula_view](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_formula_view.sql) macro for the appropriately defined table(s). In order for this command to work you must be within the root directory of your dbt project. +This bash script is intended to be used in order to automatically create the desired salesforce models via the command line within your dbt project. This bash script will generate a model file within your dbt project that contains the [sfdc_formula_view](https://github.com/fivetran/dbt_salesforce_formula_utils/blob/main/macros/sfdc_formula_view.sql) macro for the appropriately defined table(s). In order for this command to work you must be within the root directory of your dbt project. By default, the resultant models will run Option #1, but can be configured to run Options #2 and #3 by adding `full_statement_version=false` for both and the `fields_to_include` arguments for Option #3. **Usage:** ```bash From b0dbe0608bb2937dd9146fe7f820409445592e8e Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 14 Feb 2023 10:54:38 -0800 Subject: [PATCH 17/21] Apply suggestions from code review Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 36d5896..76e5b28 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ If you would like your model to generate all the formula fields related to your **Output**: All formulas for the chosen source table will be included in the resulting `select` statement. -> This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas and may be incomplete compared to Option #1. +> This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas and therefore may be incomplete compared to Option #1. ### Option 3: Generate only specified formula fields using package-made query @@ -101,7 +101,7 @@ If you would like your model to generate only a specified subset of your formula **Output**: Only formulas provided in the `fields_to_include` variable will be included in the resulting `select` statement for the chosen source table. -> This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas. +> This option makes use of the `fivetran_formula` lookup table, which requires the package to combine fields' formulas into a SQL query for each source table. This option does not support double-nested formulas and therefore may be incomplete compared to Option #1. ### Automate model creation If you have multiple models you need to create, you can also leverage the [sfdc_formula_model_automation](https://github.com/fivetran/dbt_salesforce_formula_utils#sfdc_formula_model_automationsh-source) script within this project to automatically create models locally via the command line. Below is an example command to copy and edit. From 086cfc448b8a20481b32cf2f91c06ec22df80ca0 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 16 Feb 2023 13:40:20 -0800 Subject: [PATCH 18/21] working --- macros/sfdc_formula_view.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/macros/sfdc_formula_view.sql b/macros/sfdc_formula_view.sql index 677fa15..0f847fa 100644 --- a/macros/sfdc_formula_view.sql +++ b/macros/sfdc_formula_view.sql @@ -16,8 +16,8 @@ {% if full_statement_version %} {% if using_quoted_identifiers %} {%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'), - column='"model"' if target.type in ('snowflake', 'postgres', 'redshift') else '`model`', - where=("\"object\" = '" if target.type in ('snowflake', 'postgres', 'redshift') else "`object` = '") ~ source_table ~ "'") -%} + column='"MODEL"' if target.type in ('snowflake') else '"model"' if target.type in ('postgres', 'redshift', 'snowflake') else '`model`', + where=("\"OBJECT\" = '" if target.type in ('snowflake') else "\"object\" = '" if target.type in ('postgres', 'redshift') else "`object` = '") ~ source_table ~ "'") -%} {% else %} {%- set table_results = dbt_utils.get_column_values(table=source(source_name, 'fivetran_formula_model'), column='model', where="object = '" ~ source_table ~ "'") -%} From ca39f7584ece786037ddc9a40bb7cb591a9f661d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 16 Feb 2023 15:18:19 -0800 Subject: [PATCH 19/21] Update README.md Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 76e5b28..52c755d 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ To use this dbt package, you must have the following: - A **BigQuery**, **Snowflake**, **Redshift**, or **PostgreSQL** destination. ## Step 2: Install the package -The [`sfdc_formula_view`](https://github.com/fivetran/dbt_salesforce_formula_utils#sfdc_formula_view-source) macro is intended to be leveraged within a dbt project that uses the source tables from Fivetran's Salesforce connector. To leverage the macro, you will add the below configuration to your `packages.yml` file (if you do not have a `packages.yml` file, create on in your root dbt project). +The [`sfdc_formula_view`](https://github.com/fivetran/dbt_salesforce_formula_utils#sfdc_formula_view-source) macro is intended to be leveraged within a dbt project that uses the source tables from Fivetran's Salesforce connector. To leverage the macro, you will add the below configuration to your `packages.yml` file (if you do not have a `packages.yml` file, create one in your root dbt project). > TIP: Check [dbt Hub](https://hub.getdbt.com/) for the latest installation instructions or [read the dbt docs](https://docs.getdbt.com/docs/package-management) for more information on installing packages. ```yml packages: From 1aeb77dcf1df50362f8afe610511886186fabf60 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 16 Feb 2023 15:18:32 -0800 Subject: [PATCH 20/21] Update README.md Co-authored-by: Sheri Nguyen <94874400+fivetran-sheringuyen@users.noreply.github.com> --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 52c755d..fc5b0c0 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh ".. The `sfdc_formula_view` macro has been created to support double-nested formula field references for most cases using Option #1, however, never if using Options #2 or #3. For example: - :white_check_mark: : A formula field references standard fields from the base Salesforce table. - :white_check_mark: : A formula field references another formula field that does **not** reference other formula fields. + The `sfdc_formula_view` macro may be able to support some cases of n-nested formula field references if using Option #1 above, however, never if using Options #2 or #3. For example: - 🚧 : A formula field references another formula field that references another formula field (and so on...). From 472462084b25e9664c9297df6041193457741c77 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 16 Feb 2023 15:21:44 -0800 Subject: [PATCH 21/21] bash --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fc5b0c0..44ff319 100644 --- a/README.md +++ b/README.md @@ -110,7 +110,7 @@ If you have multiple models you need to create, you can also leverage the [sfdc_ source dbt_modules/salesforce_formula_utils/sfdc_formula_model_automation.sh "../path/to/directory" "desired_table_1,desired_table_2,desired_table_infinity" ``` -**Output**: Model files for each table, populated with `{{ salesforce_formula_utils.sfdc_formula_view(source_table='table_name') }}`. By default, these models will run Option #1. To use Options #2 or #3, you will need to add `full_statement_version=false`. For Option #3, you will need to add the `fields_to_include` argument(s). +**Output**: Model files for each table, populated with `{{ salesforce_formula_utils.sfdc_formula_view(source_table='table_name') }}`. By default, these models will run Option #1. To use Options #2 or #3, you will need to manually add `full_statement_version=false` to each model file. For Option #3, you will need to add the `fields_to_include` argument(s) as well. > Note: In order for this command to work, you must currently be within the root directory of your dbt project.