diff --git a/.gitignore b/.gitignore index 05dada1..d470565 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ target/ dbt_packages/ logs/ .DS_Store +.gitconfig diff --git a/README.md b/README.md index 66fc05e..a002b93 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ vars: ```yml packages: - package: Snowflake-Labs/dbt_constraints - version: [">=0.6.0", "<0.7.0"] + version: [">=1.0.0", "<1.1.0"] # for the latest version tag. # You can also pull the latest changes from Github with the following: # - git: "https://github.com/Snowflake-Labs/dbt_constraints.git" @@ -119,66 +119,64 @@ packages: __have_ownership_priv(table_relation, verify_permissions, lookup_cache=none) ``` -## dbt_constraints Limitations +## RELY and NORELY Properties -Generally, if you don't meet a requirement, tests are still executed but the constraint is skipped rather than producing an error. +Version 1.0.0 introduces the ability to create constraints with the RELY and NORELY properties on Snowflake. Executed tests with zero failures are created with the `RELY` property. Tests with any failures will generate `NORELY` constraints and constraints will be altered to `RELY` or `NORELY` based on subsequent executions of the test. When the `always_create_constraint` feature is enabled, it is now also possible to create `NORELY` constraints using `dbt run` and then have those constraints become RELY constraints using `dbt test`. -* All models involved in a constraint must be materialized as table, incremental, snapshot, or seed. +## Determining the Constraints to Generate -* If source constraints are enabled, the source must be a table. You must also have the `OWNERSHIP` table privilege to add a constraint. For foreign keys you also need the `REFERENCES` privilege on the parent table with the primary or unique key. The package will identify when you lack these privileges on Snowflake and PostgreSQL. Oracle does not provide an easy way to look up your effective privileges so it has an exception handler and will display Oracle's error messages. +Version 1.0.0 introduces a more advanced set of criteria for selecting tests to turn into constraints. +* The test must be one of the following: `primary_key`, `unique_key`, `unique_combination_of_columns`, `unique`, `foreign_key`, `relationships`, or `not_null` +* The test executed and had zero failures (RELY) or the database has support for NORELY constraints +* The test executed (RELY/NORELY), we need the primary/unique key constraint for a foreign key, or we have the `always_create_constraint` parameter turned on. +* If you are using `build`, `run`, or `test` for only part of a project using the `--select` parameter, either the test or its model was selected to run, or the test is a primary/unique key that is needed for a selected foreign key. If a primary/unique key is created for a foreign key, and its test was not executed, the primary/unique key will be created as a NORELY constraint. +* All models involved in a constraint must **not** be a view or ephemeral materialization. Version 1.0.0 now allows custom materializations. +* If source constraints are enabled, the source must be a table. You must also have the `OWNERSHIP` table privilege to add a constraint. For foreign keys you also need the `REFERENCES` privilege on the parent table with the primary or unique key. The package will identify when you lack these privileges on Snowflake and PostgreSQL. Oracle does not provide an easy way to look up your effective privileges so it has an exception handler and will display Oracle's error messages. * All columns on constraints must be individual column names, not expressions. You can reference columns on a model that come from an expression. - -* Constraints are not created for failed tests. See how to get around this using severity and `config: always_create_constraint: true` in the next section. - * `primary_key`, `unique_key`, and `foreign_key` tests are considered first and duplicate constraints are skipped. One exception is that you will get an error if you add two different `primary_key` tests to the same model. - * Foreign keys require that the parent table have a primary key or unique key on the referenced columns. Unique keys generated from standard `unique` tests are sufficient. - * The order of columns on a foreign key test must match between the FK columns and PK columns +* Referential constraints must apply to all the rows in a table so any tests with a `config: where:`, `config: warn_if:`, or `config: fail_calc:` property will be set as `NORELY` when creating constraints. +Additional notes: * The `foreign_key` test will ignore any rows with a null column, even if only one of two columns in a compound key is null. If you also want to ensure FK columns are not null, you should add standard `not_null` tests to your model which will add not null constraints to the table. +* You may need to manually drop a primary key constraint from a table if you change the columns in the constraint. This is not necessary for table materializations or if you do a full-refresh of an incremental model. + +## Advanced: `always_create_constraint` Property -* Referential constraints must apply to all the rows in a table so any tests with a `config: where:` property will be skipped when creating constraints. See how to disable this rule using `config: always_create_constraint: true` in the next section. +There is an advanced option for Snowflake users to force a constraint to be generated even when the test was not executed. When this setting is in effect, constraints on Snowflake will have the `NORELY` property until the associated test is executed with zero failures. Snowflake does not support `NORELY` for not null constraints so those constraints will still be skipped. You activate this feature in your dbt_project.yml under the `models:` or `tests:` sections. You can set it to be true for your entire project or you can specify specific folders that should use this feature. You can also set this in a specific model's header. -## Advanced: `config: always_create_constraint: true` property +__[Caveat Emptor](https://en.wikipedia.org/wiki/Caveat_emptor):__ -There is an advanced option to force a constraint to be generated when there is a `config: where:` property or if the constraint has a threshold. The `config: always_create_constraint: true` property will override those exclusions. When this setting is in effect, you can create constraints even when you have excluded some records or have a number of failures below a threshold. If your test has a status of 'failed', it will still be skipped. Please see [dbt's documentation on how to set a threshold for failures](https://docs.getdbt.com/reference/resource-configs/severity). +* You will get an error if you try to force constraints to be generated that are enforced by your database. On Snowflake that is only a not_null constraint but on databases like Oracle, all the generated constraints are enforced. This is why, at present, only the Snowflake macros implement this feature. +* This feature can still cause unexpected query results on Snowflake due to [join elimination](https://docs.snowflake.com/en/user-guide/join-elimination). Although executing tests on Snowflake will correctly set the `RELY` or `NORELY` property based on whether the tests pass and fail, activating this feature and **skipping the execution of tests** will not cause a `RELY` constraint to become a `NORELY` constraint. A `RELY` constraint only becomes a `NORELY` constraint **if a test is executed** and has failures. If you create a `RELY` constraint by running `dbt build` and subsequently only execute `dbt run` without eventually following up with `dbt test`, you could have constraints that still have the `RELY` property but now have referential integrity issues. Snowflake users are encouraged to frequently or always execute their tests so that the `RELY` property is kept up to date. -__Caveat Emptor:__ +These are examples from a dbt_project.yml using the feature in models or tests: -* You will get an error if you try to force constraints to be generated that are enforced by your database. On Snowflake that is only a not_null constraint but on databases like Oracle, all the generated constraints are enforced. -* This feature could cause unexpected query results on Snowflake due to [join elimination](https://docs.snowflake.com/en/user-guide/join-elimination). +```yml +models: + your_project_name: + +always_create_constraint: true +tests: + your_project_name: + +always_create_constraint: true +``` -This is an example using the feature: +This is an example from a model schema.yml using the feature. Setting the property in the `config:` section of a test does not work so you should set it in the model's `config:` section. ```yml - - name: dim_duplicate_orders - description: "Test that we do not try to create PK/UK on failed tests" - columns: - - name: o_orderkey - description: "The primary key for this table" - - name: o_orderkey_seq - description: "duplicate seq column to test UK" - tests: - # This constraint should be skipped because it has failures - - dbt_constraints.primary_key: - column_name: o_orderkey - config: - severity: warn - # This constraint should be still generated because always_create_constraint=true - - dbt_constraints.unique_key: - column_name: o_orderkey - config: - warn_if: ">= 5000" - error_if: ">= 10000" - always_create_constraint: true - # This constraint should be still generated because always_create_constraint=true - - dbt_constraints.unique_key: - column_name: o_orderkey_seq - config: - severity: warn - always_create_constraint: true +version: 2 + +models: + - name: your_model_name + config: + always_create_constraint: true +``` + +This is an example of activating the feature in the header of a model: +```jinja +{{ config(always_create_constraint = true) }} ``` ## Primary Maintainers diff --git a/dbt_project.yml b/dbt_project.yml index 6055048..a32fb9f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ name: 'dbt_constraints' -version: '0.6.3' +version: '1.0.0' config-version: 2 # These macros depend on the results and graph objects in dbt >=0.19.0 diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index e484ac3..b18c802 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -50,3 +50,10 @@ seeds: +quote_columns: false +post-hook: "{{ clone_table('source_') }}" #+full_refresh: false + +tests: + +dbt_constraints_integration_tests: + #+always_create_constraint: true + # These configuration settings disable running tests or just constraints by path + # +enabled: false + #+dbt_constraints_enabled: false diff --git a/integration_tests/models/dim_part.sql b/integration_tests/models/dim_part.sql index 2905fe0..a28693c 100644 --- a/integration_tests/models/dim_part.sql +++ b/integration_tests/models/dim_part.sql @@ -2,6 +2,8 @@ All Parts Additional unique keys generated by sequence and hash */ +{{ config(always_create_constraint = true) }} + SELECT P.*, DENSE_RANK() over (order by p_partkey) as p_partkey_seq diff --git a/integration_tests/models/schema.yml b/integration_tests/models/schema.yml index 8e498a2..299e632 100644 --- a/integration_tests/models/schema.yml +++ b/integration_tests/models/schema.yml @@ -152,19 +152,17 @@ models: column_name: o_orderkey config: severity: warn - # This constraint can be generated if you uncomment always_create_constraint=true + - dbt_constraints.unique_key: column_name: o_orderkey config: warn_if: ">= 5000" error_if: ">= 10000" - # always_create_constraint: true - # This constraint can be generated if you uncomment always_create_constraint=true + - dbt_constraints.unique_key: column_name: o_orderkey_seq config: severity: warn - # always_create_constraint: true - name: fact_order_line_missing_orders description: "Test that we do not create FK on failed tests" @@ -202,6 +200,8 @@ models: - name: dim_orders_null_keys description: "All Orders" + config: + always_create_constraint: true columns: - name: o_custkey tests: @@ -215,10 +215,12 @@ models: column_name: o_orderkey config: severity: warn + # test that we still create this valid unique key - dbt_constraints.unique_key: column_name: o_orderkey_seq + - name: dim_part_supplier description: "Multi column UK" columns: diff --git a/integration_tests/models/sources.yml b/integration_tests/models/sources.yml index 12c5f34..c9828dd 100644 --- a/integration_tests/models/sources.yml +++ b/integration_tests/models/sources.yml @@ -129,7 +129,7 @@ sources: # How to validate a compound foreign key - relationships: column_name: "coalesce(cast(l_partkey as varchar(100)), '') || '~' || coalesce(cast(l_suppkey as varchar(100)), '')" - to: source('tpc_h', 'partsupp') + to: source('tpc_h', 'source_partsupp') field: "coalesce(cast(ps_partkey as varchar(100)), '') || '~' || coalesce(cast(ps_suppkey as varchar(100)), '')" # multi-column FK diff --git a/macros/create_constraints.sql b/macros/create_constraints.sql index eb9f8fe..2b029ce 100644 --- a/macros/create_constraints.sql +++ b/macros/create_constraints.sql @@ -50,23 +50,23 @@ {#- Define three create macros for PK, UK, and FK that can be overridden by DB implementations -#} -{%- macro create_primary_key(table_model, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} - {{ return(adapter.dispatch('create_primary_key', 'dbt_constraints')(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache)) }} +{%- macro create_primary_key(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} + {{ return(adapter.dispatch('create_primary_key', 'dbt_constraints')(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause)) }} {%- endmacro -%} -{%- macro create_unique_key(table_model, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} - {{ return(adapter.dispatch('create_unique_key', 'dbt_constraints')(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache)) }} +{%- macro create_unique_key(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} + {{ return(adapter.dispatch('create_unique_key', 'dbt_constraints')(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause)) }} {%- endmacro -%} -{%- macro create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache) -%} - {{ return(adapter.dispatch('create_foreign_key', 'dbt_constraints')(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache)) }} +{%- macro create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} + {{ return(adapter.dispatch('create_foreign_key', 'dbt_constraints')(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause)) }} {%- endmacro -%} -{%- macro create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache) -%} - {{ return(adapter.dispatch('create_not_null', 'dbt_constraints')(table_relation, column_names, verify_permissions, quote_columns, lookup_cache)) }} +{%- macro create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} + {{ return(adapter.dispatch('create_not_null', 'dbt_constraints')(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause)) }} {%- endmacro -%} @@ -92,6 +92,17 @@ {%- endmacro -%} +{#- Define macro for whether a DB implementation has implemented logic for RELY and NORELY constraints -#} + +{%- macro adapter_supports_rely_norely(test_name) -%} + {{ return(adapter.dispatch('adapter_supports_rely_norely', 'dbt_constraints')(test_name)) }} +{%- endmacro -%} + +{#- By default, we assume DB implementations have NOT implemented logic for RELY and NORELY constraints -#} +{%- macro default__adapter_supports_rely_norely(test_name) -%} + {{ return(false) }} +{%- endmacro -%} + {#- Override dbt's truncate_relation macro to allow us to create adapter specific versions that drop constraints -#} @@ -160,201 +171,300 @@ {%- endmacro -%} +{#- This macro checks if a test or its model is selected -#} +{%- macro test_selected(test_model) -%} + {%- if test_model.unique_id in selected_resources -%} + {{ return("TEST_SELECTED") }} + {%- endif -%} + {%- if lists_intersect(test_model.depends_on.nodes, selected_resources) -%} + {{ return("MODEL_SELECTED") }} + {%- endif -%} -{#- This macro is called internally and passed which constraint types to create. -#} -{%- macro create_constraints_by_type(constraint_types, quote_columns, lookup_cache) -%} + {#- Check if a PK/UK should be created because it is referenced by a selected FK -#} + {%- if test_model.test_metadata + and test_model.test_metadata.name in ("primary_key", "unique_key", "unique_combination_of_columns", "unique") -%} + {%- set pk_test_args = test_model.test_metadata.kwargs -%} + {%- set pk_test_columns = [] -%} + {%- if pk_test_args.column_names -%} + {%- set pk_test_columns = pk_test_args.column_names -%} + {%- elif pk_test_args.combination_of_columns -%} + {%- set pk_test_columns = pk_test_args.combination_of_columns -%} + {%- elif pk_test_args.column_name -%} + {%- set pk_test_columns = [pk_test_args.column_name] -%} + {%- endif -%} + {%- for fk_model in graph.nodes.values() | selectattr("resource_type", "equalto", "test") + if fk_model.test_metadata + and fk_model.test_metadata.name in ("foreign_key", "relationships") + and lists_intersect(test_model.depends_on.nodes, fk_model.depends_on.nodes) + and ( fk_model.unique_id in selected_resources + or lists_intersect(fk_model.depends_on.nodes, selected_resources) ) -%} + {%- set fk_test_args = fk_model.test_metadata.kwargs -%} + {%- set fk_test_columns = [] -%} + {%- if fk_test_args.pk_column_names -%} + {%- set fk_test_columns = fk_test_args.pk_column_names -%} + {%- elif fk_test_args.pk_column_name -%} + {%- set fk_test_columns = [fk_test_args.pk_column_name] -%} + {%- elif fk_test_args.field -%} + {%- set fk_test_columns = [fk_test_args.field] -%} + {%- endif -%} + {%- if column_list_matches(pk_test_columns, fk_test_columns) -%} + {{ return("PK_UK_FOR_SELECTED_FK") }} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {{ return(none) }} +{%- endmacro -%} + + +{#- This macro that checks if a test has results and whether there were errors -#} +{%- macro lookup_should_rely(test_model) -%} + {%- if test_model.config.where + or test_model.config.warn_if != "!= 0" + or test_model.config.fail_calc != "count(*)" -%} + {#- Set NORELY if there is a condition on the test -#} + {{ return('NORELY') }} + {%- endif -%} - {#- Loop through the results and find all tests that passed and match the constraint_types -#} - {#- Issue #2: added condition that the where config must be empty -#} {%- for res in results if res.node.config.materialized == "test" - and res.status in ("pass", "warn") - and res.node.test_metadata - and res.node.test_metadata.name is in( constraint_types ) - and ( res.failures == 0 or - res.node.config.get("always_create_constraint", false) ) - and ( res.node.config.where is none or - res.node.config.get("always_create_constraint", false) ) - and res.node.config.get("dbt_constraints_enabled", true) -%} - - {%- set test_model = res.node -%} - {%- set test_parameters = test_model.test_metadata.kwargs -%} - {% set ns = namespace(verify_permissions=false) %} + and res.node.unique_id == test_model.unique_id -%} + {%- if res.failures > 0 -%} + {#- Set NORELY if there is a test failure -#} + {{ return('NORELY') }} + {%- elif res.failures == 0 -%} + {#- Set RELY if there are 0 failures -#} + {{ return('RELY') }} + {%- endif -%} + {%- endfor -%} + {{ return('') }} +{%- endmacro -%} - {#- Find the table models that are referenced by this test. - These models must be physical tables and cannot be sources -#} - {%- set table_models = [] -%} - {%- for node in graph.nodes.values() | selectattr("unique_id", "in", test_model.depends_on.nodes) - if node.resource_type in ( ( "model", "snapshot", "seed") ) - if node.config.materialized in( ("table", "incremental", "snapshot", "seed") ) -%} - {#- Append to our list of models &or snapshots for this test -#} - {%- do table_models.append(node) -%} +{#- This macro that checks if a test or its model has always_create_constraint set -#} +{%- macro should_always_create_constraint(test_model) -%} + {%- if test_model.config.get("always_create_constraint", false) == true -%} + {{ return(true) }} + {%- endif -%} + {%- for table_node in test_model.depends_on.nodes -%} + {%- for node in graph.nodes.values() | selectattr("unique_id", "equalto", table_node) + if node.config.get("always_create_constraint", false) == true -%} + {{ return(true) }} + {%- endfor -%} + {%- endfor -%} - {% endfor %} + {{ return(false) }} +{%- endmacro -%} - {#- Check if we allow constraints on sources overall and for this specific type of constraint -#} - {%- if var('dbt_constraints_sources_enabled', false) and ( - ( var('dbt_constraints_sources_pk_enabled', false) and test_model.test_metadata.name in("primary_key") ) - or ( var('dbt_constraints_sources_uk_enabled', false) and test_model.test_metadata.name in("unique_key", "unique_combination_of_columns", "unique") ) - or ( var('dbt_constraints_sources_fk_enabled', false) and test_model.test_metadata.name in("foreign_key", "relationships") ) - or ( var('dbt_constraints_sources_nn_enabled', false) and test_model.test_metadata.name in("not_null") ) - ) -%} - {%- for node in graph.sources.values() - | selectattr("resource_type", "equalto", "source") - | selectattr("unique_id", "in", test_model.depends_on.nodes) -%} - {%- do node.update({'alias': node.alias or node.name }) -%} - {#- Append to our list of models for this test -#} - {%- do table_models.append(node) -%} - {#- If we are using a sources, we will need to verify permissions -#} - {%- set ns.verify_permissions = true -%} +{#- This macro is called internally and passed which constraint types to create. -#} +{%- macro create_constraints_by_type(constraint_types, quote_columns, lookup_cache) -%} - {%- endfor -%} - {%- endif -%} + {#- Loop through the metadata and find all tests that match the constraint_types -#} + {%- for test_model in graph.nodes.values() | selectattr("resource_type", "equalto", "test") + if test_model.test_metadata + and test_model.test_metadata.name is in( constraint_types ) + and test_model.config.enabled + and test_model.config.get("dbt_constraints_enabled", true) -%} + {%- set test_parameters = test_model.test_metadata.kwargs -%} + {%- set test_name = test_model.test_metadata.name -%} + {%- set selected = test_selected(test_model) -%} + + {#- We can shortcut additional tests if the constraint was not selected -#} + {%- if selected is not none -%} + {#- rely_clause clause will be RELY if a test passed, NORELY if it failed, and '' if it was skipped -#} + {%- set rely_clause = lookup_should_rely(test_model) -%} + {%- set always_create_constraint = should_always_create_constraint(test_model) -%} + {%- else -%} + {%- set rely_clause = '' -%} + {%- set always_create_constraint = false -%} + {%- endif -%} - {#- We only create PK/UK if there is one model referenced by the test - and if all the columns exist as physical columns on the table -#} - {%- if 1 == table_models|count - and test_model.test_metadata.name in("primary_key", "unique_key", "unique_combination_of_columns", "unique") -%} - - {# Attempt to identify a parameter we can use for the column names #} - {%- set column_names = [] -%} - {%- if test_parameters.column_names -%} - {%- set column_names = test_parameters.column_names -%} - {%- elif test_parameters.combination_of_columns -%} - {%- set column_names = test_parameters.combination_of_columns -%} - {%- elif test_parameters.column_name -%} - {%- set column_names = [test_parameters.column_name] -%} - {%- else -%} - {{ exceptions.raise_compiler_error( - "`column_names` or `column_name` parameter missing for primary/unique key constraint on table: '" ~ table_models[0].name - ) }} - {%- endif -%} + {#- Create constraints that: + - Either the test or its model was selected to run, including PK/UK for FK + - Passed the test (RELY) or the database supports NORELY constraints + - We ran the test (RELY/NORELY) or we need the constraint for a FK + or we have the always_create_constraint parameter turned on -#} + {%- if selected is not none + and ( rely_clause == 'RELY' + or dbt_constraints.adapter_supports_rely_norely(test_name) == true ) + and ( rely_clause in('RELY', 'NORELY') + or selected == "PK_UK_FOR_SELECTED_FK" + or always_create_constraint == true ) -%} + + {% set ns = namespace(verify_permissions=false) %} + {%- set table_models = [] -%} + + {#- Find the table models that are referenced by this test. -#} + {%- for table_node in test_model.depends_on.nodes -%} + {%- for node in graph.nodes.values() | selectattr("unique_id", "equalto", table_node) + if node.config.get("materialized", "other") not in ("view", "ephemeral") + and ( node.resource_type in ("model", "snapshot", "seed") + or ( node.resource_type == "source" and var('dbt_constraints_sources_enabled', false) + and ( ( var('dbt_constraints_sources_pk_enabled', false) and test_name in("primary_key") ) + or ( var('dbt_constraints_sources_uk_enabled', false) and test_name in("unique_key", "unique_combination_of_columns", "unique") ) + or ( var('dbt_constraints_sources_fk_enabled', false) and test_name in("foreign_key", "relationships") ) + or ( var('dbt_constraints_sources_nn_enabled', false) and test_name in("not_null") ) ) + ) ) -%} - {%- set table_relation = api.Relation.create( - database=table_models[0].database, - schema=table_models[0].schema, - identifier=table_models[0].alias ) -%} - {%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%} - {%- if test_model.test_metadata.name == "primary_key" -%} - {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache) -%} - {%- do dbt_constraints.create_primary_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache) -%} + {%- do node.update({'alias': node.alias or node.name }) -%} + {#- Append to our list of models for this test -#} + {%- do table_models.append(node) -%} + {%- if node.resource_type == "source" + or node.config.get("materialized", "other") not in ("table", "incremental", "snapshot", "seed") -%} + {#- If we are using a sources or custom materializations, we will need to verify permissions -#} + {%- set ns.verify_permissions = true -%} + {%- endif -%} + + {% endfor %} + {% endfor %} + + {#- We only create PK/UK if there is one model referenced by the test + and if all the columns exist as physical columns on the table -#} + {%- if 1 == table_models|count + and test_name in("primary_key", "unique_key", "unique_combination_of_columns", "unique") -%} + + {# Attempt to identify a parameter we can use for the column names #} + {%- set column_names = [] -%} + {%- if test_parameters.column_names -%} + {%- set column_names = test_parameters.column_names -%} + {%- elif test_parameters.combination_of_columns -%} + {%- set column_names = test_parameters.combination_of_columns -%} + {%- elif test_parameters.column_name -%} + {%- set column_names = [test_parameters.column_name] -%} {%- else -%} - {%- do dbt_constraints.create_unique_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache) -%} + {{ exceptions.raise_compiler_error( + "`column_names` or `column_name` parameter missing for primary/unique key constraint on table: '" ~ table_models[0].name + ) }} {%- endif -%} - {%- else -%} - {%- do log("Skipping primary/unique key because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%} - {%- endif -%} - {#- We only create FK if there are two models referenced by the test - and if all the columns exist as physical columns on the tables -#} - {%- elif 2 == table_models|count - and test_model.test_metadata.name in( "foreign_key", "relationships") -%} - - {%- set fk_model = none -%} - {%- set pk_model = none -%} - {%- set fk_model_names = modules.re.findall( "(models|snapshots|seeds)\W+(\w+)" , test_model.file_key_name) -%} - {%- set fk_source_names = modules.re.findall( "source\W+(\w+)\W+(\w+)" , test_parameters.model) -%} - - {%- if 1 == fk_model_names | count -%} - {%- set fk_model = table_models | selectattr("name", "equalto", fk_model_names[0][1]) | first -%} - {%- set pk_model = table_models | rejectattr("name", "equalto", fk_model_names[0][1]) | first -%} - {%- elif 1 == fk_source_names | count -%} - {%- if table_models[0].source_name == fk_source_names[0][0] and table_models[0].name == fk_source_names[0][1] -%} - {%- set fk_model = table_models[0] -%} - {%- set pk_model = table_models[1] -%} + {%- set table_relation = api.Relation.create( + database=table_models[0].database, + schema=table_models[0].schema, + identifier=table_models[0].alias ) -%} + {%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%} + {%- if test_name == "primary_key" -%} + {%- if dbt_constraints.adapter_supports_rely_norely("not_null") == true -%} + {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%} + {%- endif -%} + {%- do dbt_constraints.create_primary_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} + {%- else -%} + {%- do dbt_constraints.create_unique_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} + {%- endif -%} {%- else -%} - {%- set fk_model = table_models[1] -%} - {%- set pk_model = table_models[0] -%} + {%- do log("Skipping primary/unique key because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%} {%- endif -%} - {%- endif -%} - {# {%- set fk_model_name = test_model.file_key_name |replace("models.", "") -%} #} - - {%- if fk_model and pk_model -%} - - {%- set fk_table_relation = api.Relation.create( - database=fk_model.database, - schema=fk_model.schema, - identifier=fk_model.alias) -%} - - {%- set pk_table_relation = api.Relation.create( - database=pk_model.database, - schema=pk_model.schema, - identifier=pk_model.alias) -%} - - {# Attempt to identify parameters we can use for the column names #} - {%- set pk_column_names = [] -%} - {%- if test_parameters.pk_column_names -%} - {%- set pk_column_names = test_parameters.pk_column_names -%} - {%- elif test_parameters.field -%} - {%- set pk_column_names = [test_parameters.field] -%} - {%- elif test_parameters.pk_column_name -%} - {%- set pk_column_names = [test_parameters.pk_column_name] -%} - {%- else -%} - {{ exceptions.raise_compiler_error( - "`pk_column_names`, `pk_column_name`, or `field` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters - ) }} + + {#- We only create FK if there are two models referenced by the test + and if all the columns exist as physical columns on the tables -#} + {%- elif 2 == table_models|count + and test_name in( "foreign_key", "relationships") -%} + + {%- set fk_model = none -%} + {%- set pk_model = none -%} + {%- set fk_model_names = modules.re.findall( "(models|snapshots|seeds)\W+(\w+)" , test_model.file_key_name) -%} + {%- set fk_source_names = modules.re.findall( "source\W+(\w+)\W+(\w+)" , test_parameters.model) -%} + + {%- if 1 == fk_model_names | count -%} + {%- set fk_model = table_models | selectattr("name", "equalto", fk_model_names[0][1]) | first -%} + {%- set pk_model = table_models | rejectattr("name", "equalto", fk_model_names[0][1]) | first -%} + {%- elif 1 == fk_source_names | count -%} + {%- if table_models[0].source_name == fk_source_names[0][0] and table_models[0].name == fk_source_names[0][1] -%} + {%- set fk_model = table_models[0] -%} + {%- set pk_model = table_models[1] -%} + {%- else -%} + {%- set fk_model = table_models[1] -%} + {%- set pk_model = table_models[0] -%} + {%- endif -%} + {%- endif -%} + {# {%- set fk_model_name = test_model.file_key_name |replace("models.", "") -%} #} + + {%- if fk_model and pk_model -%} + + {%- set fk_table_relation = api.Relation.create( + database=fk_model.database, + schema=fk_model.schema, + identifier=fk_model.alias) -%} + + {%- set pk_table_relation = api.Relation.create( + database=pk_model.database, + schema=pk_model.schema, + identifier=pk_model.alias) -%} + + {# Attempt to identify parameters we can use for the column names #} + {%- set pk_column_names = [] -%} + {%- if test_parameters.pk_column_names -%} + {%- set pk_column_names = test_parameters.pk_column_names -%} + {%- elif test_parameters.field -%} + {%- set pk_column_names = [test_parameters.field] -%} + {%- elif test_parameters.pk_column_name -%} + {%- set pk_column_names = [test_parameters.pk_column_name] -%} + {%- else -%} + {{ exceptions.raise_compiler_error( + "`pk_column_names`, `pk_column_name`, or `field` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters + ) }} + {%- endif -%} + + {%- set fk_column_names = [] -%} + {%- if test_parameters.fk_column_names -%} + {%- set fk_column_names = test_parameters.fk_column_names -%} + {%- elif test_parameters.column_name -%} + {%- set fk_column_names = [test_parameters.column_name] -%} + {%- elif test_parameters.fk_column_name -%} + {%- set fk_column_names = [test_parameters.fk_column_name] -%} + {%- else -%} + {{ exceptions.raise_compiler_error( + "`fk_column_names`, `fk_column_name`, or `column_name` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters + ) }} + {%- endif -%} + + {%- if not dbt_constraints.table_columns_all_exist(pk_table_relation, pk_column_names, lookup_cache) -%} + {%- do log("Skipping foreign key because a physical column was not found on the pk table: " ~ pk_model.name ~ " " ~ pk_column_names, info=true) -%} + {%- elif not dbt_constraints.table_columns_all_exist(fk_table_relation, fk_column_names, lookup_cache) -%} + {%- do log("Skipping foreign key because a physical column was not found on the fk table: " ~ fk_model.name ~ " " ~ fk_column_names, info=true) -%} + {%- else -%} + {%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} + {%- endif -%} + {%- else -%} + {%- do log("Skipping foreign key because a we couldn't find the child table: model=" ~ fk_model_names ~ " or source=" ~ fk_source_names, info=true) -%} {%- endif -%} - {%- set fk_column_names = [] -%} - {%- if test_parameters.fk_column_names -%} - {%- set fk_column_names = test_parameters.fk_column_names -%} - {%- elif test_parameters.column_name -%} - {%- set fk_column_names = [test_parameters.column_name] -%} - {%- elif test_parameters.fk_column_name -%} - {%- set fk_column_names = [test_parameters.fk_column_name] -%} - {%- else -%} + {#- We only create NN if there is one model referenced by the test + and if all the columns exist as physical columns on the table -#} + {%- elif 1 == table_models|count + and test_name in("not_null") -%} + + {# Attempt to identify a parameter we can use for the column names #} + {%- set column_names = [] -%} + {%- if test_parameters.column_names -%} + {%- set column_names = test_parameters.column_names -%} + {%- elif test_parameters.combination_of_columns -%} + {%- set column_names = test_parameters.combination_of_columns -%} + {%- elif test_parameters.column_name -%} + {%- set column_names = [test_parameters.column_name] -%} + {%- else -%} {{ exceptions.raise_compiler_error( - "`fk_column_names`, `fk_column_name`, or `column_name` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters + "`column_names` or `column_name` parameter missing for not null constraint on table: '" ~ table_models[0].name ) }} {%- endif -%} - {%- if not dbt_constraints.table_columns_all_exist(pk_table_relation, pk_column_names, lookup_cache) -%} - {%- do log("Skipping foreign key because a physical column was not found on the pk table: " ~ pk_model.name ~ " " ~ pk_column_names, info=true) -%} - {%- elif not dbt_constraints.table_columns_all_exist(fk_table_relation, fk_column_names, lookup_cache) -%} - {%- do log("Skipping foreign key because a physical column was not found on the fk table: " ~ fk_model.name ~ " " ~ fk_column_names, info=true) -%} + {%- set table_relation = api.Relation.create( + database=table_models[0].database, + schema=table_models[0].schema, + identifier=table_models[0].alias ) -%} + + {%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%} + {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- else -%} - {%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache) -%} + {%- do log("Skipping not null constraint because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%} {%- endif -%} - {%- else -%} - {%- do log("Skipping foreign key because a we couldn't find the child table: model=" ~ fk_model_names ~ " or source=" ~ fk_source_names, info=true) -%} - {%- endif -%} - {#- We only create NN if there is one model referenced by the test - and if all the columns exist as physical columns on the table -#} - {%- elif 1 == table_models|count - and test_model.test_metadata.name in("not_null") -%} - - {# Attempt to identify a parameter we can use for the column names #} - {%- set column_names = [] -%} - {%- if test_parameters.column_names -%} - {%- set column_names = test_parameters.column_names -%} - {%- elif test_parameters.combination_of_columns -%} - {%- set column_names = test_parameters.combination_of_columns -%} - {%- elif test_parameters.column_name -%} - {%- set column_names = [test_parameters.column_name] -%} - {%- else -%} - {{ exceptions.raise_compiler_error( - "`column_names` or `column_name` parameter missing for not null constraint on table: '" ~ table_models[0].name - ) }} {%- endif -%} - - {%- set table_relation = api.Relation.create( - database=table_models[0].database, - schema=table_models[0].schema, - identifier=table_models[0].alias ) -%} - - {%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%} - {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache) -%} - {%- else -%} - {%- do log("Skipping not null constraint because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%} - {%- endif -%} - {%- endif -%} + {%- endfor -%} {%- endmacro -%} @@ -408,3 +518,11 @@ {{ return(false) }} {%- endif -%} {%- endmacro -%} + +{# This macro allows us to compare two lists to see if they intersect #} +{%- macro lists_intersect(listA, listB) -%} + {%- for valueFromA in listA if valueFromA in listB -%} + {{ return(true) }} + {% endfor %} + {{ return(false) }} +{%- endmacro -%} diff --git a/macros/oracle__create_constraints.sql b/macros/oracle__create_constraints.sql index 6eac607..1e2238d 100644 --- a/macros/oracle__create_constraints.sql +++ b/macros/oracle__create_constraints.sql @@ -1,5 +1,5 @@ {# Oracle specific implementation to create a primary key #} -{%- macro oracle__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro oracle__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} {%- if constraint_name|length > 30 %} @@ -42,7 +42,7 @@ END; {# Oracle specific implementation to create a unique key #} -{%- macro oracle__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro oracle__create_unique_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} {%- if constraint_name|length > 30 %} @@ -85,7 +85,7 @@ END; {# Oracle specific implementation to create a foreign key #} -{%- macro oracle__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache) -%} +{%- macro oracle__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} {%- if constraint_name|length > 30 %} @@ -131,7 +131,7 @@ END; {%- endmacro -%} {# Oracle specific implementation to create a not null constraint #} -{%- macro oracle__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache) -%} +{%- macro oracle__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- set columns_list = dbt_constraints.get_quoted_column_list(column_names, quote_columns) -%} {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions) -%} diff --git a/macros/postgres__create_constraints.sql b/macros/postgres__create_constraints.sql index a002a07..66d0dd2 100644 --- a/macros/postgres__create_constraints.sql +++ b/macros/postgres__create_constraints.sql @@ -1,5 +1,5 @@ {# PostgreSQL specific implementation to create a primary key #} -{%- macro postgres__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro postgres__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} {%- if constraint_name|length > 63 %} @@ -36,7 +36,7 @@ {# PostgreSQL specific implementation to create a unique key #} -{%- macro postgres__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro postgres__create_unique_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} {%- if constraint_name|length > 63 %} @@ -71,7 +71,7 @@ {%- endmacro -%} {# PostgreSQL specific implementation to create a not null constraint #} -{%- macro postgres__create_not_null(table_relation, column_names, verify_permissions, quote_columns=false, lookup_cache=none) -%} +{%- macro postgres__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- set columns_list = dbt_constraints.get_quoted_column_list(column_names, quote_columns) -%} {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} @@ -93,7 +93,7 @@ {%- endmacro -%} {# PostgreSQL specific implementation to create a foreign key #} -{%- macro postgres__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns=true, constraint_name=none, lookup_cache=none) -%} +{%- macro postgres__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} {%- if constraint_name|length > 63 %} diff --git a/macros/redshift__create_constraints.sql b/macros/redshift__create_constraints.sql index a06f6c1..ba280ae 100644 --- a/macros/redshift__create_constraints.sql +++ b/macros/redshift__create_constraints.sql @@ -1,5 +1,5 @@ {# Redshift specific implementation to create a primary key #} -{%- macro redshift__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro redshift__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} {%- if constraint_name|length > 127 %} @@ -36,7 +36,7 @@ {# Redshift specific implementation to create a unique key #} -{%- macro redshift__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro redshift__create_unique_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} {%- if constraint_name|length > 127 %} @@ -71,14 +71,14 @@ {%- endmacro -%} {# Redshift specific implementation to create a not null constraint #} -{%- macro redshift__create_not_null(table_relation, column_names, verify_permissions, quote_columns=false, lookup_cache=none) -%} +{%- macro redshift__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- set columns_list = dbt_constraints.get_quoted_column_list(column_names, quote_columns) -%} {%- do log("Skipping not null constraint for " ~ columns_list | join(", ") ~ " in " ~ table_relation ~ " because ALTER COLUMN SET NOT NULL is not supported", info=true) -%} {%- endmacro -%} {# Redshift specific implementation to create a foreign key #} -{%- macro redshift__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns=true, constraint_name=none, lookup_cache=none) -%} +{%- macro redshift__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} {%- if constraint_name|length > 127 %} diff --git a/macros/snowflake__create_constraints.sql b/macros/snowflake__create_constraints.sql index 9026f87..35ed439 100644 --- a/macros/snowflake__create_constraints.sql +++ b/macros/snowflake__create_constraints.sql @@ -1,21 +1,47 @@ +{#- Snowflake supports RELY and NORELY constraints for PK, UK, FK but not not_null -#} +{%- macro snowflake__adapter_supports_rely_norely(test_name) -%} + {%- if test_name in ( + 'primary_key', + 'unique_key', + 'unique_combination_of_columns', + 'unique', + 'foreign_key', + 'relationships') -%} + {{ return(true) }} + {%- else -%} + {{ return(false) }} + {%- endif -%} +{%- endmacro -%} + + {# Snowflake specific implementation to create a primary key #} -{%- macro snowflake__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro snowflake__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} {#- Check that the table does not already have this PK/UK -#} -{%- if not dbt_constraints.unique_constraint_exists(table_relation, column_names, lookup_cache) -%} +{%- set existing_constraint = dbt_constraints.unique_constraint_exists(table_relation, column_names, lookup_cache) -%} +{%- if constraint_name == existing_constraint -%} + {%- do set_rely_norely(table_relation, constraint_name, lookup_cache.unique_keys[table_relation][constraint_name].rely, rely_clause) -%} + {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: + { "constraint_name": constraint_name, + "columns": column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} +{%- elif none == existing_constraint -%} {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} + {%- set rely_clause = 'NORELY' if rely_clause == '' else rely_clause -%} {%- set query -%} - ALTER TABLE {{ table_relation }} ADD CONSTRAINT {{ constraint_name }} PRIMARY KEY ( {{ columns_csv }} ) RELY + ALTER TABLE {{ table_relation }} ADD CONSTRAINT {{ constraint_name }} PRIMARY KEY ( {{ columns_csv }} ) {{ rely_clause }} {%- endset -%} - {%- do log("Creating primary key: " ~ constraint_name, info=true) -%} + {%- do log("Creating primary key: " ~ constraint_name ~ " " ~ rely_clause, info=true) -%} {%- do run_query(query) -%} {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: column_names} }) -%} - + {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: + { "constraint_name": constraint_name, + "columns": column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} {%- else -%} {%- do log("Skipping " ~ constraint_name ~ " because of insufficient privileges: " ~ table_relation, info=true) -%} {%- endif -%} @@ -30,22 +56,33 @@ {# Snowflake specific implementation to create a unique key #} -{%- macro snowflake__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro snowflake__create_unique_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} {#- Check that the table does not already have this PK/UK -#} -{%- if not dbt_constraints.unique_constraint_exists(table_relation, column_names, lookup_cache) -%} +{%- set existing_constraint = dbt_constraints.unique_constraint_exists(table_relation, column_names, lookup_cache) -%} +{%- if constraint_name == existing_constraint -%} + {%- do set_rely_norely(table_relation, constraint_name, lookup_cache.unique_keys[table_relation][constraint_name].rely, rely_clause) -%} + {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: + { "constraint_name": constraint_name, + "columns": column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} +{%- elif none == existing_constraint -%} {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} + {%- set rely_clause = 'NORELY' if rely_clause == '' else rely_clause -%} {%- set query -%} - ALTER TABLE {{ table_relation }} ADD CONSTRAINT {{ constraint_name }} UNIQUE ( {{ columns_csv }} ) RELY + ALTER TABLE {{ table_relation }} ADD CONSTRAINT {{ constraint_name }} UNIQUE ( {{ columns_csv }} ) {{ rely_clause }} {%- endset -%} - {%- do log("Creating unique key: " ~ constraint_name, info=true) -%} + {%- do log("Creating unique key: " ~ constraint_name ~ " " ~ rely_clause, info=true) -%} {%- do run_query(query) -%} {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: column_names} }) -%} + {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: + { "constraint_name": constraint_name, + "columns": column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} {%- else -%} {%- do log("Skipping " ~ constraint_name ~ " because of insufficient privileges: " ~ table_relation, info=true) -%} @@ -60,24 +97,36 @@ {# Snowflake specific implementation to create a foreign key #} -{%- macro snowflake__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache) -%} +{%- macro snowflake__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} {%- set fk_columns_csv = dbt_constraints.get_quoted_column_csv(fk_column_names, quote_columns) -%} {%- set pk_columns_csv = dbt_constraints.get_quoted_column_csv(pk_column_names, quote_columns) -%} + {#- Check that the PK table has a PK or UK -#} -{%- if dbt_constraints.unique_constraint_exists(pk_table_relation, pk_column_names, lookup_cache) -%} +{%- if none != dbt_constraints.unique_constraint_exists(pk_table_relation, pk_column_names, lookup_cache) -%} {#- Check if the table already has this foreign key -#} - {%- if not dbt_constraints.foreign_key_exists(fk_table_relation, fk_column_names, lookup_cache) -%} + {%- set existing_constraint = dbt_constraints.foreign_key_exists(fk_table_relation, fk_column_names, lookup_cache) -%} + {%- if constraint_name == existing_constraint -%} + {%- do set_rely_norely(fk_table_relation, constraint_name, lookup_cache.foreign_keys[fk_table_relation][constraint_name].rely, rely_clause) -%} + {%- do lookup_cache.foreign_keys.update({fk_table_relation: {constraint_name: + {"constraint_name": constraint_name, + "columns": fk_column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} + {%- elif none == existing_constraint -%} {%- if dbt_constraints.have_ownership_priv(fk_table_relation, verify_permissions, lookup_cache) and dbt_constraints.have_references_priv(pk_table_relation, verify_permissions, lookup_cache) -%} + {%- set rely_clause = 'NORELY' if rely_clause == '' else rely_clause -%} {%- set query -%} - ALTER TABLE {{ fk_table_relation }} ADD CONSTRAINT {{ constraint_name }} FOREIGN KEY ( {{ fk_columns_csv }} ) REFERENCES {{ pk_table_relation }} ( {{ pk_columns_csv }} ) RELY + ALTER TABLE {{ fk_table_relation }} ADD CONSTRAINT {{ constraint_name }} FOREIGN KEY ( {{ fk_columns_csv }} ) REFERENCES {{ pk_table_relation }} ( {{ pk_columns_csv }} ) {{ rely_clause }} {%- endset -%} - {%- do log("Creating foreign key: " ~ constraint_name ~ " referencing " ~ pk_table_relation.identifier ~ " " ~ pk_column_names, info=true) -%} + {%- do log("Creating foreign key: " ~ constraint_name ~ " referencing " ~ pk_table_relation.identifier ~ " " ~ pk_column_names ~ " " ~ rely_clause, info=true) -%} {%- do run_query(query) -%} {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.foreign_keys.update({fk_table_relation: {constraint_name: fk_column_names} }) -%} + {%- do lookup_cache.foreign_keys.update({fk_table_relation: {constraint_name: + {"constraint_name": constraint_name, + "columns": fk_column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} {%- else -%} {%- do log("Skipping " ~ constraint_name ~ " because of insufficient privileges: " ~ fk_table_relation ~ " referencing " ~ pk_table_relation, info=true) -%} @@ -95,57 +144,73 @@ {# Snowflake specific implementation to create a not null constraint #} -{%- macro snowflake__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache) -%} +{%- macro snowflake__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} +{%- if not rely_clause == 'RELY' -%} + {%- do log("Skipping not null constraint for " ~ column_names | join(", ") ~ " in " ~ table_relation ~ " because Snowflake does not support NORELY for not null constraints.", info=true) -%} + {{ return(false) }} +{%- endif -%} {%- set existing_not_null_col = lookup_cache.not_null_col[table_relation] -%} {%- set columns_to_change = [] -%} {%- for column_name in column_names if column_name not in existing_not_null_col -%} -{%- do columns_to_change.append(column_name) -%} -{%- do existing_not_null_col.append(column_name) -%} + {%- do columns_to_change.append(column_name) -%} + {%- do existing_not_null_col.append(column_name) -%} {%- endfor -%} {%- if columns_to_change|count > 0 -%} -{%- set columns_list = dbt_constraints.get_quoted_column_list(columns_to_change, quote_columns) -%} + {%- set columns_list = dbt_constraints.get_quoted_column_list(columns_to_change, quote_columns) -%} -{%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} + {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} - {%- set modify_statements= [] -%} - {%- for column in columns_list -%} - {%- set modify_statements = modify_statements.append( "COLUMN " ~ column ~ " SET NOT NULL" ) -%} - {%- endfor -%} - {%- set modify_statement_csv = modify_statements | join(", ") -%} - {%- set query -%} - ALTER TABLE {{ table_relation }} MODIFY {{ modify_statement_csv }}; - {%- endset -%} - {%- do log("Creating not null constraint for: " ~ columns_to_change | join(", ") ~ " in " ~ table_relation, info=true) -%} - {%- do run_query(query) -%} - {#- Add this constraint to the lookup cache -#} - {%- set constraint_key = table_relation.identifier ~ "_" ~ columns_to_change|join('_') ~ "_NN" -%} - {%- do lookup_cache.not_null_col.update({table_relation: existing_not_null_col }) -%} + {%- set modify_statements= [] -%} + {%- for column in columns_list -%} + {%- set modify_statements = modify_statements.append( "COLUMN " ~ column ~ " SET NOT NULL" ) -%} + {%- endfor -%} + {%- set modify_statement_csv = modify_statements | join(", ") -%} + {%- set query -%} + ALTER TABLE {{ table_relation }} MODIFY {{ modify_statement_csv }}; + {%- endset -%} + {%- do log("Creating not null constraint for: " ~ columns_to_change | join(", ") ~ " in " ~ table_relation ~ " " ~ rely_clause, info=true) -%} + {%- do run_query(query) -%} + {#- Add this constraint to the lookup cache -#} + {%- set constraint_key = table_relation.identifier ~ "_" ~ columns_to_change|join('_') ~ "_NN" -%} + {%- do lookup_cache.not_null_col.update({table_relation: existing_not_null_col }) -%} - {%- else -%} - {%- do log("Skipping not null constraint for " ~ columns_to_change | join(", ") ~ " in " ~ table_relation ~ " because of insufficient privileges: " ~ table_relation, info=true) -%} - {%- endif -%} {%- else -%} - {%- do log("Skipping not null constraint for " ~ column_names | join(", ") ~ " in " ~ table_relation ~ " because all columns are already not null", info=false) -%} + {%- do log("Skipping not null constraint for " ~ columns_to_change | join(", ") ~ " in " ~ table_relation ~ " because of insufficient privileges: " ~ table_relation, info=true) -%} {%- endif -%} +{%- else -%} + {%- do log("Skipping not null constraint for " ~ column_names | join(", ") ~ " in " ~ table_relation ~ " because all columns are already not null", info=false) -%} +{%- endif -%} + {%- endmacro -%} +{#- This macro alters constraints to use RELY or NORELY based on failed and passed tests -#} +{%- macro set_rely_norely(table_relation, constraint_name, constraint_rely, rely_clause) -%} + {%- if ( rely_clause == 'NORELY' and constraint_rely == 'true' ) + or ( rely_clause == 'RELY' and constraint_rely == 'false' ) -%} + {%- set query -%} + ALTER TABLE {{ table_relation }} MODIFY CONSTRAINT {{ constraint_name }} {{ rely_clause }} + {%- endset -%} + {%- do log("Updating constraint: " ~ constraint_name ~ " " ~ rely_clause, info=true) -%} + {%- do run_query(query) -%} + {%- endif -%} +{%- endmacro -%} + {#- This macro is used in create macros to avoid duplicate PK/UK constraints and to skip FK where no PK/UK constraint exists on the parent table -#} {%- macro snowflake__unique_constraint_exists(table_relation, column_names, lookup_cache) -%} - {#- Check if we can find this constraint in the lookup cache -#} {%- if table_relation in lookup_cache.unique_keys -%} -{%- set cached_unique_keys = lookup_cache.unique_keys[table_relation] -%} -{%- for cached_columns in cached_unique_keys.values() -%} -{%- if dbt_constraints.column_list_matches(cached_columns, column_names ) -%} -{%- do log("Found UK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} -{{ return(true) }} -{%- endif -%} -{% endfor %} + {%- set cached_unique_keys = lookup_cache.unique_keys[table_relation] -%} + {%- for cached_val in cached_unique_keys.values() -%} + {%- if dbt_constraints.column_list_matches(cached_val.columns, column_names ) -%} + {%- do log("Found UK key: " ~ table_relation ~ " " ~ cached_val.columns ~ " " ~ cached_val.rely, info=false) -%} + {{ return(cached_val.constraint_name) }} + {%- endif -%} + {% endfor %} {%- endif -%} {%- set lookup_query -%} @@ -153,38 +218,46 @@ SHOW UNIQUE KEYS IN TABLE {{ table_relation }} {%- endset -%} {%- set constraint_list = run_query(lookup_query) -%} {%- if constraint_list.columns["column_name"].values() | count > 0 -%} - {%- for constraint in constraint_list.group_by("constraint_name") -%} - {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.unique_keys.update({table_relation: {constraint.key_name: constraint.columns["column_name"].values()} }) -%} - {% endfor %} - {%- for constraint in constraint_list.group_by("constraint_name") -%} - {%- if dbt_constraints.column_list_matches(constraint.columns["column_name"].values(), column_names ) -%} - {%- do log("Found UK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} - {{ return(true) }} - {%- endif -%} - {% endfor %} - {%- endif -%} + {%- for constraint in constraint_list.group_by("constraint_name") -%} + {%- set existing_constraint_name = (constraint.columns["constraint_name"].values() | first) -%} + {%- set existing_columns = constraint.columns["column_name"].values() -%} + {%- set existing_rely = (constraint.columns["rely"].values() | first) -%} + {#- Add this constraint to the lookup cache -#} + {%- do lookup_cache.unique_keys.update({table_relation: {existing_constraint_name: + { "constraint_name": existing_constraint_name, + "columns": existing_columns, + "rely": existing_rely } } }) -%} + {%- if dbt_constraints.column_list_matches(existing_columns, column_names ) -%} + {%- do log("Found UK key: " ~ existing_constraint_name ~ " " ~ table_relation ~ " " ~ column_names ~ " " ~ existing_rely, info=false) -%} + {{ return(existing_constraint_name) }} + {%- endif -%} + {% endfor %} +{%- endif -%} {%- set lookup_query -%} SHOW PRIMARY KEYS IN TABLE {{ table_relation }} {%- endset -%} {%- set constraint_list = run_query(lookup_query) -%} {%- if constraint_list.columns["column_name"].values() | count > 0 -%} - {%- for constraint in constraint_list.group_by("constraint_name") -%} - {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.unique_keys.update({table_relation: {constraint.key_name: constraint.columns["column_name"].values()} }) -%} - {% endfor %} - {%- for constraint in constraint_list.group_by("constraint_name") -%} - {%- if dbt_constraints.column_list_matches(constraint.columns["column_name"].values(), column_names ) -%} - {%- do log("Found PK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} - {{ return(true) }} - {%- endif -%} - {% endfor %} - {%- endif -%} + {%- for constraint in constraint_list.group_by("constraint_name") -%} + {%- set existing_constraint_name = (constraint.columns["constraint_name"].values() | first) -%} + {%- set existing_columns = constraint.columns["column_name"].values() -%} + {%- set existing_rely = (constraint.columns["rely"].values() | first) -%} + {#- Add this constraint to the lookup cache -#} + {%- do lookup_cache.unique_keys.update({table_relation: {existing_constraint_name: + { "constraint_name": existing_constraint_name, + "columns": existing_columns, + "rely": existing_rely } } }) -%} + {%- if dbt_constraints.column_list_matches(existing_columns, column_names ) -%} + {%- do log("Found PK key: " ~ existing_constraint_name ~ " " ~ table_relation ~ " " ~ column_names ~ " " ~ existing_rely, info=false) -%} + {{ return(existing_constraint_name) }} + {%- endif -%} + {% endfor %} +{%- endif -%} {#- If we get this far then the table does not have either constraint -#} {%- do log("No PK/UK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} -{{ return(false) }} +{{ return(none) }} {%- endmacro -%} @@ -194,13 +267,13 @@ SHOW PRIMARY KEYS IN TABLE {{ table_relation }} {#- Check if we can find this constraint in the lookup cache -#} {%- if table_relation in lookup_cache.foreign_keys -%} -{%- set cached_foreign_keys = lookup_cache.foreign_keys[table_relation] -%} -{%- for cached_columns in cached_foreign_keys.values() -%} -{%- if dbt_constraints.column_list_matches(cached_columns, column_names ) -%} -{%- do log("Found FK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} -{{ return(true) }} -{%- endif -%} -{% endfor %} + {%- set cached_foreign_keys = lookup_cache.foreign_keys[table_relation] -%} + {%- for cached_val in cached_foreign_keys.values() -%} + {%- if dbt_constraints.column_list_matches(cached_val.columns, column_names ) -%} + {%- do log("Found FK key: " ~ table_relation ~ " " ~ cached_val.constraint_name ~ " " ~ column_names ~ " " ~ cached_val.rely, info=false) -%} + {{ return(cached_val.constraint_name) }} + {%- endif -%} + {% endfor %} {%- endif -%} {%- set lookup_query -%} @@ -208,21 +281,25 @@ SHOW IMPORTED KEYS IN TABLE {{ table_relation }} {%- endset -%} {%- set constraint_list = run_query(lookup_query) -%} {%- if constraint_list.columns["fk_column_name"].values() | count > 0 -%} - {%- for constraint in constraint_list.group_by("fk_name") -%} - {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.foreign_keys.update({table_relation: {constraint.key_name: constraint.columns["fk_column_name"].values()} }) -%} - {% endfor %} - {%- for constraint in constraint_list.group_by("fk_name") -%} - {%- if dbt_constraints.column_list_matches(constraint.columns["fk_column_name"].values(), column_names ) -%} - {%- do log("Found FK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} - {{ return(true) }} - {%- endif -%} - {% endfor %} - {%- endif -%} + {%- for constraint in constraint_list.group_by("fk_name") -%} + {%- set existing_constraint_name = (constraint.columns["fk_name"].values() | first) -%} + {%- set existing_columns = constraint.columns["fk_column_name"].values() -%} + {%- set existing_rely = (constraint.columns["rely"].values() | first) -%} + {#- Add this constraint to the lookup cache -#} + {%- do lookup_cache.foreign_keys.update({table_relation: {existing_constraint_name: + { "constraint_name": existing_constraint_name, + "columns": existing_columns, + "rely": existing_rely } } }) -%} + {%- if dbt_constraints.column_list_matches(existing_columns, column_names ) -%} + {%- do log("Found FK key: " ~ table_relation ~ " " ~ existing_constraint_name ~ " " ~ column_names ~ " " ~ existing_rely, info=false) -%} + {{ return(existing_constraint_name) }} + {%- endif -%} + {% endfor %} +{%- endif -%} {#- If we get this far then the table does not have this constraint -#} {%- do log("No FK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} -{{ return(false) }} +{{ return(none) }} {%- endmacro -%} diff --git a/macros/vertica__create_constraints.sql b/macros/vertica__create_constraints.sql index 1900da3..5d5edd2 100644 --- a/macros/vertica__create_constraints.sql +++ b/macros/vertica__create_constraints.sql @@ -1,5 +1,5 @@ {# Vertica specific implementation to create a primary key #} -{%- macro vertica__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro vertica__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} @@ -28,7 +28,7 @@ {# Vertica specific implementation to create a unique key #} -{%- macro vertica__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro vertica__create_unique_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} @@ -57,7 +57,7 @@ {# Vertica specific implementation to create a foreign key #} -{%- macro vertica__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache) -%} +{%- macro vertica__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} {%- set fk_columns_csv = dbt_constraints.get_quoted_column_csv(fk_column_names, quote_columns) -%} {%- set pk_columns_csv = dbt_constraints.get_quoted_column_csv(pk_column_names, quote_columns) -%} @@ -91,7 +91,7 @@ {# Vertica specific implementation to create a not null constraint #} -{%- macro vertica__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache) -%} +{%- macro vertica__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- set existing_not_null_col = lookup_cache.not_null_col[table_relation] -%}