From 72bc96e1b7bc522811320d6fdbbdc63f2c1f0018 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 15 Jul 2024 18:02:05 -0400 Subject: [PATCH 01/16] Fixed missing source in relationship --- integration_tests/models/sources.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/models/sources.yml b/integration_tests/models/sources.yml index 12c5f34..c9828dd 100644 --- a/integration_tests/models/sources.yml +++ b/integration_tests/models/sources.yml @@ -129,7 +129,7 @@ sources: # How to validate a compound foreign key - relationships: column_name: "coalesce(cast(l_partkey as varchar(100)), '') || '~' || coalesce(cast(l_suppkey as varchar(100)), '')" - to: source('tpc_h', 'partsupp') + to: source('tpc_h', 'source_partsupp') field: "coalesce(cast(ps_partkey as varchar(100)), '') || '~' || coalesce(cast(ps_suppkey as varchar(100)), '')" # multi-column FK From 188ad1dba4483b061fa3754123d7e2c04ac843af Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 15 Jul 2024 18:02:35 -0400 Subject: [PATCH 02/16] Removed reference to always_create_constraints --- integration_tests/models/schema.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/integration_tests/models/schema.yml b/integration_tests/models/schema.yml index 8e498a2..8cf264c 100644 --- a/integration_tests/models/schema.yml +++ b/integration_tests/models/schema.yml @@ -152,19 +152,17 @@ models: column_name: o_orderkey config: severity: warn - # This constraint can be generated if you uncomment always_create_constraint=true + - dbt_constraints.unique_key: column_name: o_orderkey config: warn_if: ">= 5000" error_if: ">= 10000" - # always_create_constraint: true - # This constraint can be generated if you uncomment always_create_constraint=true + - dbt_constraints.unique_key: column_name: o_orderkey_seq config: severity: warn - # always_create_constraint: true - name: fact_order_line_missing_orders description: "Test that we do not create FK on failed tests" From bc8267315fb6595c151731905355530fb1670543 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 15 Jul 2024 18:04:14 -0400 Subject: [PATCH 03/16] Added unused rely_clause to create macros --- macros/oracle__create_constraints.sql | 8 ++++---- macros/postgres__create_constraints.sql | 8 ++++---- macros/redshift__create_constraints.sql | 8 ++++---- macros/vertica__create_constraints.sql | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/macros/oracle__create_constraints.sql b/macros/oracle__create_constraints.sql index 6eac607..1e2238d 100644 --- a/macros/oracle__create_constraints.sql +++ b/macros/oracle__create_constraints.sql @@ -1,5 +1,5 @@ {# Oracle specific implementation to create a primary key #} -{%- macro oracle__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro oracle__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} {%- if constraint_name|length > 30 %} @@ -42,7 +42,7 @@ END; {# Oracle specific implementation to create a unique key #} -{%- macro oracle__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro oracle__create_unique_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} {%- if constraint_name|length > 30 %} @@ -85,7 +85,7 @@ END; {# Oracle specific implementation to create a foreign key #} -{%- macro oracle__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache) -%} +{%- macro oracle__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} {%- if constraint_name|length > 30 %} @@ -131,7 +131,7 @@ END; {%- endmacro -%} {# Oracle specific implementation to create a not null constraint #} -{%- macro oracle__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache) -%} +{%- macro oracle__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- set columns_list = dbt_constraints.get_quoted_column_list(column_names, quote_columns) -%} {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions) -%} diff --git a/macros/postgres__create_constraints.sql b/macros/postgres__create_constraints.sql index a002a07..66d0dd2 100644 --- a/macros/postgres__create_constraints.sql +++ b/macros/postgres__create_constraints.sql @@ -1,5 +1,5 @@ {# PostgreSQL specific implementation to create a primary key #} -{%- macro postgres__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro postgres__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} {%- if constraint_name|length > 63 %} @@ -36,7 +36,7 @@ {# PostgreSQL specific implementation to create a unique key #} -{%- macro postgres__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro postgres__create_unique_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} {%- if constraint_name|length > 63 %} @@ -71,7 +71,7 @@ {%- endmacro -%} {# PostgreSQL specific implementation to create a not null constraint #} -{%- macro postgres__create_not_null(table_relation, column_names, verify_permissions, quote_columns=false, lookup_cache=none) -%} +{%- macro postgres__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- set columns_list = dbt_constraints.get_quoted_column_list(column_names, quote_columns) -%} {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} @@ -93,7 +93,7 @@ {%- endmacro -%} {# PostgreSQL specific implementation to create a foreign key #} -{%- macro postgres__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns=true, constraint_name=none, lookup_cache=none) -%} +{%- macro postgres__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} {%- if constraint_name|length > 63 %} diff --git a/macros/redshift__create_constraints.sql b/macros/redshift__create_constraints.sql index a06f6c1..ba280ae 100644 --- a/macros/redshift__create_constraints.sql +++ b/macros/redshift__create_constraints.sql @@ -1,5 +1,5 @@ {# Redshift specific implementation to create a primary key #} -{%- macro redshift__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro redshift__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} {%- if constraint_name|length > 127 %} @@ -36,7 +36,7 @@ {# Redshift specific implementation to create a unique key #} -{%- macro redshift__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro redshift__create_unique_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} {%- if constraint_name|length > 127 %} @@ -71,14 +71,14 @@ {%- endmacro -%} {# Redshift specific implementation to create a not null constraint #} -{%- macro redshift__create_not_null(table_relation, column_names, verify_permissions, quote_columns=false, lookup_cache=none) -%} +{%- macro redshift__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- set columns_list = dbt_constraints.get_quoted_column_list(column_names, quote_columns) -%} {%- do log("Skipping not null constraint for " ~ columns_list | join(", ") ~ " in " ~ table_relation ~ " because ALTER COLUMN SET NOT NULL is not supported", info=true) -%} {%- endmacro -%} {# Redshift specific implementation to create a foreign key #} -{%- macro redshift__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns=true, constraint_name=none, lookup_cache=none) -%} +{%- macro redshift__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} {%- if constraint_name|length > 127 %} diff --git a/macros/vertica__create_constraints.sql b/macros/vertica__create_constraints.sql index 1900da3..5d5edd2 100644 --- a/macros/vertica__create_constraints.sql +++ b/macros/vertica__create_constraints.sql @@ -1,5 +1,5 @@ {# Vertica specific implementation to create a primary key #} -{%- macro vertica__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro vertica__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} @@ -28,7 +28,7 @@ {# Vertica specific implementation to create a unique key #} -{%- macro vertica__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro vertica__create_unique_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} @@ -57,7 +57,7 @@ {# Vertica specific implementation to create a foreign key #} -{%- macro vertica__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache) -%} +{%- macro vertica__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} {%- set fk_columns_csv = dbt_constraints.get_quoted_column_csv(fk_column_names, quote_columns) -%} {%- set pk_columns_csv = dbt_constraints.get_quoted_column_csv(pk_column_names, quote_columns) -%} @@ -91,7 +91,7 @@ {# Vertica specific implementation to create a not null constraint #} -{%- macro vertica__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache) -%} +{%- macro vertica__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- set existing_not_null_col = lookup_cache.not_null_col[table_relation] -%} From f70555fc6284c0f55a6472df62c6045b3ea9c77a Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 15 Jul 2024 18:06:18 -0400 Subject: [PATCH 04/16] Implemented Snowflake logic to create and switch NORELY and RELY constraints based on test results --- macros/snowflake__create_constraints.sql | 227 ++++++++++++++--------- 1 file changed, 138 insertions(+), 89 deletions(-) diff --git a/macros/snowflake__create_constraints.sql b/macros/snowflake__create_constraints.sql index 9026f87..93e7764 100644 --- a/macros/snowflake__create_constraints.sql +++ b/macros/snowflake__create_constraints.sql @@ -1,21 +1,27 @@ {# Snowflake specific implementation to create a primary key #} -{%- macro snowflake__create_primary_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro snowflake__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} {#- Check that the table does not already have this PK/UK -#} -{%- if not dbt_constraints.unique_constraint_exists(table_relation, column_names, lookup_cache) -%} +{%- set existing_constraint = dbt_constraints.unique_constraint_exists(table_relation, column_names, lookup_cache) -%} +{%- if constraint_name == existing_constraint -%} + {%- do set_rely_norely(table_relation, constraint_name, lookup_cache.unique_keys[table_relation][constraint_name].rely, rely_clause) -%} +{%- elif none == existing_constraint -%} {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} + {%- set rely_clause = 'NORELY' if rely_clause == '' else rely_clause -%} {%- set query -%} - ALTER TABLE {{ table_relation }} ADD CONSTRAINT {{ constraint_name }} PRIMARY KEY ( {{ columns_csv }} ) RELY + ALTER TABLE {{ table_relation }} ADD CONSTRAINT {{ constraint_name }} PRIMARY KEY ( {{ columns_csv }} ) {{ rely_clause }} {%- endset -%} - {%- do log("Creating primary key: " ~ constraint_name, info=true) -%} + {%- do log("Creating primary key: " ~ constraint_name ~ " " ~ rely_clause, info=true) -%} {%- do run_query(query) -%} {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: column_names} }) -%} - + {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: + { "constraint_name": constraint_name, + "columns": column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} {%- else -%} {%- do log("Skipping " ~ constraint_name ~ " because of insufficient privileges: " ~ table_relation, info=true) -%} {%- endif -%} @@ -30,22 +36,29 @@ {# Snowflake specific implementation to create a unique key #} -{%- macro snowflake__create_unique_key(table_relation, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} +{%- macro snowflake__create_unique_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_UK") | upper -%} {%- set columns_csv = dbt_constraints.get_quoted_column_csv(column_names, quote_columns) -%} {#- Check that the table does not already have this PK/UK -#} -{%- if not dbt_constraints.unique_constraint_exists(table_relation, column_names, lookup_cache) -%} +{%- set existing_constraint = dbt_constraints.unique_constraint_exists(table_relation, column_names, lookup_cache) -%} +{%- if constraint_name == existing_constraint -%} + {%- do set_rely_norely(table_relation, constraint_name, lookup_cache.unique_keys[table_relation][constraint_name].rely, rely_clause) -%} +{%- elif none == existing_constraint -%} {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} + {%- set rely_clause = 'NORELY' if rely_clause == '' else rely_clause -%} {%- set query -%} - ALTER TABLE {{ table_relation }} ADD CONSTRAINT {{ constraint_name }} UNIQUE ( {{ columns_csv }} ) RELY + ALTER TABLE {{ table_relation }} ADD CONSTRAINT {{ constraint_name }} UNIQUE ( {{ columns_csv }} ) {{ rely_clause }} {%- endset -%} - {%- do log("Creating unique key: " ~ constraint_name, info=true) -%} + {%- do log("Creating unique key: " ~ constraint_name ~ " " ~ rely_clause, info=true) -%} {%- do run_query(query) -%} {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: column_names} }) -%} + {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: + { "constraint_name": constraint_name, + "columns": column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} {%- else -%} {%- do log("Skipping " ~ constraint_name ~ " because of insufficient privileges: " ~ table_relation, info=true) -%} @@ -60,24 +73,32 @@ {# Snowflake specific implementation to create a foreign key #} -{%- macro snowflake__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache) -%} +{%- macro snowflake__create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or fk_table_relation.identifier ~ "_" ~ fk_column_names|join('_') ~ "_FK") | upper -%} {%- set fk_columns_csv = dbt_constraints.get_quoted_column_csv(fk_column_names, quote_columns) -%} {%- set pk_columns_csv = dbt_constraints.get_quoted_column_csv(pk_column_names, quote_columns) -%} + {#- Check that the PK table has a PK or UK -#} -{%- if dbt_constraints.unique_constraint_exists(pk_table_relation, pk_column_names, lookup_cache) -%} +{%- if none != dbt_constraints.unique_constraint_exists(pk_table_relation, pk_column_names, lookup_cache) -%} {#- Check if the table already has this foreign key -#} - {%- if not dbt_constraints.foreign_key_exists(fk_table_relation, fk_column_names, lookup_cache) -%} + {%- set existing_constraint = dbt_constraints.foreign_key_exists(fk_table_relation, fk_column_names, lookup_cache) -%} + {%- if constraint_name == existing_constraint -%} + {%- do set_rely_norely(fk_table_relation, constraint_name, lookup_cache.foreign_keys[fk_table_relation][constraint_name].rely, rely_clause) -%} + {%- elif none == existing_constraint -%} {%- if dbt_constraints.have_ownership_priv(fk_table_relation, verify_permissions, lookup_cache) and dbt_constraints.have_references_priv(pk_table_relation, verify_permissions, lookup_cache) -%} + {%- set rely_clause = 'NORELY' if rely_clause == '' else rely_clause -%} {%- set query -%} - ALTER TABLE {{ fk_table_relation }} ADD CONSTRAINT {{ constraint_name }} FOREIGN KEY ( {{ fk_columns_csv }} ) REFERENCES {{ pk_table_relation }} ( {{ pk_columns_csv }} ) RELY + ALTER TABLE {{ fk_table_relation }} ADD CONSTRAINT {{ constraint_name }} FOREIGN KEY ( {{ fk_columns_csv }} ) REFERENCES {{ pk_table_relation }} ( {{ pk_columns_csv }} ) {{ rely_clause }} {%- endset -%} - {%- do log("Creating foreign key: " ~ constraint_name ~ " referencing " ~ pk_table_relation.identifier ~ " " ~ pk_column_names, info=true) -%} + {%- do log("Creating foreign key: " ~ constraint_name ~ " referencing " ~ pk_table_relation.identifier ~ " " ~ pk_column_names ~ " " ~ rely_clause, info=true) -%} {%- do run_query(query) -%} {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.foreign_keys.update({fk_table_relation: {constraint_name: fk_column_names} }) -%} + {%- do lookup_cache.foreign_keys.update({fk_table_relation: {constraint_name: + {"constraint_name": constraint_name, + "columns": fk_column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} {%- else -%} {%- do log("Skipping " ~ constraint_name ~ " because of insufficient privileges: " ~ fk_table_relation ~ " referencing " ~ pk_table_relation, info=true) -%} @@ -95,57 +116,73 @@ {# Snowflake specific implementation to create a not null constraint #} -{%- macro snowflake__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache) -%} +{%- macro snowflake__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} +{%- if not rely_clause == 'RELY' -%} + {%- do log("Skipping not null constraint for " ~ column_names | join(", ") ~ " in " ~ table_relation ~ " because Snowflake does not support NORELY for not null constraints.", info=false) -%} + {{ return(false) }} +{%- endif -%} {%- set existing_not_null_col = lookup_cache.not_null_col[table_relation] -%} {%- set columns_to_change = [] -%} {%- for column_name in column_names if column_name not in existing_not_null_col -%} -{%- do columns_to_change.append(column_name) -%} -{%- do existing_not_null_col.append(column_name) -%} + {%- do columns_to_change.append(column_name) -%} + {%- do existing_not_null_col.append(column_name) -%} {%- endfor -%} {%- if columns_to_change|count > 0 -%} -{%- set columns_list = dbt_constraints.get_quoted_column_list(columns_to_change, quote_columns) -%} + {%- set columns_list = dbt_constraints.get_quoted_column_list(columns_to_change, quote_columns) -%} -{%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} + {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} - {%- set modify_statements= [] -%} - {%- for column in columns_list -%} - {%- set modify_statements = modify_statements.append( "COLUMN " ~ column ~ " SET NOT NULL" ) -%} - {%- endfor -%} - {%- set modify_statement_csv = modify_statements | join(", ") -%} - {%- set query -%} - ALTER TABLE {{ table_relation }} MODIFY {{ modify_statement_csv }}; - {%- endset -%} - {%- do log("Creating not null constraint for: " ~ columns_to_change | join(", ") ~ " in " ~ table_relation, info=true) -%} - {%- do run_query(query) -%} - {#- Add this constraint to the lookup cache -#} - {%- set constraint_key = table_relation.identifier ~ "_" ~ columns_to_change|join('_') ~ "_NN" -%} - {%- do lookup_cache.not_null_col.update({table_relation: existing_not_null_col }) -%} + {%- set modify_statements= [] -%} + {%- for column in columns_list -%} + {%- set modify_statements = modify_statements.append( "COLUMN " ~ column ~ " SET NOT NULL" ) -%} + {%- endfor -%} + {%- set modify_statement_csv = modify_statements | join(", ") -%} + {%- set query -%} + ALTER TABLE {{ table_relation }} MODIFY {{ modify_statement_csv }}; + {%- endset -%} + {%- do log("Creating not null constraint for: " ~ columns_to_change | join(", ") ~ " in " ~ table_relation ~ " " ~ rely_clause, info=true) -%} + {%- do run_query(query) -%} + {#- Add this constraint to the lookup cache -#} + {%- set constraint_key = table_relation.identifier ~ "_" ~ columns_to_change|join('_') ~ "_NN" -%} + {%- do lookup_cache.not_null_col.update({table_relation: existing_not_null_col }) -%} - {%- else -%} - {%- do log("Skipping not null constraint for " ~ columns_to_change | join(", ") ~ " in " ~ table_relation ~ " because of insufficient privileges: " ~ table_relation, info=true) -%} - {%- endif -%} {%- else -%} - {%- do log("Skipping not null constraint for " ~ column_names | join(", ") ~ " in " ~ table_relation ~ " because all columns are already not null", info=false) -%} + {%- do log("Skipping not null constraint for " ~ columns_to_change | join(", ") ~ " in " ~ table_relation ~ " because of insufficient privileges: " ~ table_relation, info=true) -%} {%- endif -%} +{%- else -%} + {%- do log("Skipping not null constraint for " ~ column_names | join(", ") ~ " in " ~ table_relation ~ " because all columns are already not null", info=false) -%} +{%- endif -%} + {%- endmacro -%} +{#- This macro alters constraints to use RELY or NORELY based on failed and passed tests -#} +{%- macro set_rely_norely(table_relation, constraint_name, constraint_rely, rely_clause) -%} + {%- if ( rely_clause == 'NORELY' and constraint_rely == 'true' ) + or ( rely_clause == 'RELY' and constraint_rely == 'false' ) -%} + {%- set query -%} + ALTER TABLE {{ table_relation }} MODIFY CONSTRAINT {{ constraint_name }} {{ rely_clause }} + {%- endset -%} + {%- do log("Updating constraint: " ~ constraint_name ~ " " ~ rely_clause, info=true) -%} + {%- do run_query(query) -%} + {%- endif -%} +{%- endmacro -%} + {#- This macro is used in create macros to avoid duplicate PK/UK constraints and to skip FK where no PK/UK constraint exists on the parent table -#} {%- macro snowflake__unique_constraint_exists(table_relation, column_names, lookup_cache) -%} - {#- Check if we can find this constraint in the lookup cache -#} {%- if table_relation in lookup_cache.unique_keys -%} -{%- set cached_unique_keys = lookup_cache.unique_keys[table_relation] -%} -{%- for cached_columns in cached_unique_keys.values() -%} -{%- if dbt_constraints.column_list_matches(cached_columns, column_names ) -%} -{%- do log("Found UK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} -{{ return(true) }} -{%- endif -%} -{% endfor %} + {%- set cached_unique_keys = lookup_cache.unique_keys[table_relation] -%} + {%- for cached_val in cached_unique_keys.values() -%} + {%- if dbt_constraints.column_list_matches(cached_val.columns, column_names ) -%} + {%- do log("Found UK key: " ~ table_relation ~ " " ~ cached_val.columns ~ " " ~ cached_val.rely, info=false) -%} + {{ return(cached_val.constraint_name) }} + {%- endif -%} + {% endfor %} {%- endif -%} {%- set lookup_query -%} @@ -153,38 +190,46 @@ SHOW UNIQUE KEYS IN TABLE {{ table_relation }} {%- endset -%} {%- set constraint_list = run_query(lookup_query) -%} {%- if constraint_list.columns["column_name"].values() | count > 0 -%} - {%- for constraint in constraint_list.group_by("constraint_name") -%} - {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.unique_keys.update({table_relation: {constraint.key_name: constraint.columns["column_name"].values()} }) -%} - {% endfor %} - {%- for constraint in constraint_list.group_by("constraint_name") -%} - {%- if dbt_constraints.column_list_matches(constraint.columns["column_name"].values(), column_names ) -%} - {%- do log("Found UK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} - {{ return(true) }} - {%- endif -%} - {% endfor %} - {%- endif -%} + {%- for constraint in constraint_list.group_by("constraint_name") -%} + {%- set existing_constraint_name = (constraint.columns["constraint_name"].values() | first) -%} + {%- set existing_columns = constraint.columns["column_name"].values() -%} + {%- set existing_rely = (constraint.columns["rely"].values() | first) -%} + {#- Add this constraint to the lookup cache -#} + {%- do lookup_cache.unique_keys.update({table_relation: {existing_constraint_name: + { "constraint_name": existing_constraint_name, + "columns": existing_columns, + "rely": existing_rely } } }) -%} + {%- if dbt_constraints.column_list_matches(existing_columns, column_names ) -%} + {%- do log("Found UK key: " ~ existing_constraint_name ~ " " ~ table_relation ~ " " ~ column_names ~ " " ~ existing_rely, info=false) -%} + {{ return(existing_constraint_name) }} + {%- endif -%} + {% endfor %} +{%- endif -%} {%- set lookup_query -%} SHOW PRIMARY KEYS IN TABLE {{ table_relation }} {%- endset -%} {%- set constraint_list = run_query(lookup_query) -%} {%- if constraint_list.columns["column_name"].values() | count > 0 -%} - {%- for constraint in constraint_list.group_by("constraint_name") -%} - {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.unique_keys.update({table_relation: {constraint.key_name: constraint.columns["column_name"].values()} }) -%} - {% endfor %} - {%- for constraint in constraint_list.group_by("constraint_name") -%} - {%- if dbt_constraints.column_list_matches(constraint.columns["column_name"].values(), column_names ) -%} - {%- do log("Found PK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} - {{ return(true) }} - {%- endif -%} - {% endfor %} - {%- endif -%} + {%- for constraint in constraint_list.group_by("constraint_name") -%} + {%- set existing_constraint_name = (constraint.columns["constraint_name"].values() | first) -%} + {%- set existing_columns = constraint.columns["column_name"].values() -%} + {%- set existing_rely = (constraint.columns["rely"].values() | first) -%} + {#- Add this constraint to the lookup cache -#} + {%- do lookup_cache.unique_keys.update({table_relation: {existing_constraint_name: + { "constraint_name": existing_constraint_name, + "columns": existing_columns, + "rely": existing_rely } } }) -%} + {%- if dbt_constraints.column_list_matches(existing_columns, column_names ) -%} + {%- do log("Found PK key: " ~ existing_constraint_name ~ " " ~ table_relation ~ " " ~ column_names ~ " " ~ existing_rely, info=false) -%} + {{ return(existing_constraint_name) }} + {%- endif -%} + {% endfor %} +{%- endif -%} {#- If we get this far then the table does not have either constraint -#} {%- do log("No PK/UK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} -{{ return(false) }} +{{ return(none) }} {%- endmacro -%} @@ -194,13 +239,13 @@ SHOW PRIMARY KEYS IN TABLE {{ table_relation }} {#- Check if we can find this constraint in the lookup cache -#} {%- if table_relation in lookup_cache.foreign_keys -%} -{%- set cached_foreign_keys = lookup_cache.foreign_keys[table_relation] -%} -{%- for cached_columns in cached_foreign_keys.values() -%} -{%- if dbt_constraints.column_list_matches(cached_columns, column_names ) -%} -{%- do log("Found FK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} -{{ return(true) }} -{%- endif -%} -{% endfor %} + {%- set cached_foreign_keys = lookup_cache.foreign_keys[table_relation] -%} + {%- for cached_val in cached_foreign_keys.values() -%} + {%- if dbt_constraints.column_list_matches(cached_val.columns, column_names ) -%} + {%- do log("Found FK key: " ~ table_relation ~ " " ~ cached_val.constraint_name ~ " " ~ column_names ~ " " ~ cached_val.rely, info=false) -%} + {{ return(cached_val.constraint_name) }} + {%- endif -%} + {% endfor %} {%- endif -%} {%- set lookup_query -%} @@ -208,21 +253,25 @@ SHOW IMPORTED KEYS IN TABLE {{ table_relation }} {%- endset -%} {%- set constraint_list = run_query(lookup_query) -%} {%- if constraint_list.columns["fk_column_name"].values() | count > 0 -%} - {%- for constraint in constraint_list.group_by("fk_name") -%} - {#- Add this constraint to the lookup cache -#} - {%- do lookup_cache.foreign_keys.update({table_relation: {constraint.key_name: constraint.columns["fk_column_name"].values()} }) -%} - {% endfor %} - {%- for constraint in constraint_list.group_by("fk_name") -%} - {%- if dbt_constraints.column_list_matches(constraint.columns["fk_column_name"].values(), column_names ) -%} - {%- do log("Found FK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} - {{ return(true) }} - {%- endif -%} - {% endfor %} - {%- endif -%} + {%- for constraint in constraint_list.group_by("fk_name") -%} + {%- set existing_constraint_name = (constraint.columns["fk_name"].values() | first) -%} + {%- set existing_columns = constraint.columns["fk_column_name"].values() -%} + {%- set existing_rely = (constraint.columns["rely"].values() | first) -%} + {#- Add this constraint to the lookup cache -#} + {%- do lookup_cache.foreign_keys.update({table_relation: {existing_constraint_name: + { "constraint_name": existing_constraint_name, + "columns": existing_columns, + "rely": existing_rely } } }) -%} + {%- if dbt_constraints.column_list_matches(existing_columns, column_names ) -%} + {%- do log("Found FK key: " ~ table_relation ~ " " ~ existing_constraint_name ~ " " ~ column_names ~ " " ~ existing_rely, info=false) -%} + {{ return(existing_constraint_name) }} + {%- endif -%} + {% endfor %} +{%- endif -%} {#- If we get this far then the table does not have this constraint -#} {%- do log("No FK key: " ~ table_relation ~ " " ~ column_names, info=false) -%} -{{ return(false) }} +{{ return(none) }} {%- endmacro -%} From 9542cc7d93bdcc318d856a2d7813831ca09cddfd Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 15 Jul 2024 18:07:12 -0400 Subject: [PATCH 05/16] implemented RELY and NORELY constraints --- macros/create_constraints.sql | 130 +++++++++++++++++++--------------- 1 file changed, 72 insertions(+), 58 deletions(-) diff --git a/macros/create_constraints.sql b/macros/create_constraints.sql index eb9f8fe..cd2de28 100644 --- a/macros/create_constraints.sql +++ b/macros/create_constraints.sql @@ -50,23 +50,23 @@ {#- Define three create macros for PK, UK, and FK that can be overridden by DB implementations -#} -{%- macro create_primary_key(table_model, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} - {{ return(adapter.dispatch('create_primary_key', 'dbt_constraints')(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache)) }} +{%- macro create_primary_key(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} + {{ return(adapter.dispatch('create_primary_key', 'dbt_constraints')(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause)) }} {%- endmacro -%} -{%- macro create_unique_key(table_model, column_names, verify_permissions, quote_columns=false, constraint_name=none, lookup_cache=none) -%} - {{ return(adapter.dispatch('create_unique_key', 'dbt_constraints')(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache)) }} +{%- macro create_unique_key(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} + {{ return(adapter.dispatch('create_unique_key', 'dbt_constraints')(table_model, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause)) }} {%- endmacro -%} -{%- macro create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache) -%} - {{ return(adapter.dispatch('create_foreign_key', 'dbt_constraints')(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache)) }} +{%- macro create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} + {{ return(adapter.dispatch('create_foreign_key', 'dbt_constraints')(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause)) }} {%- endmacro -%} -{%- macro create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache) -%} - {{ return(adapter.dispatch('create_not_null', 'dbt_constraints')(table_relation, column_names, verify_permissions, quote_columns, lookup_cache)) }} +{%- macro create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} + {{ return(adapter.dispatch('create_not_null', 'dbt_constraints')(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause)) }} {%- endmacro -%} @@ -160,65 +160,77 @@ {%- endmacro -%} +{#- This macro that checks if a test has results and whether there were errors -#} +{%- macro lookup_should_rely(test_unique_id) -%} + {%- for res in results + if res.node.config.materialized == "test" + and res.node.unique_id == test_unique_id -%} + {%- if res.failures > 0 -%} + {#- Set NORELY if there is a test failure -#} + {{ return('NORELY') }} + {%- elif res.failures == 0 -%} + {#- Set RELY if there are 0 failures -#} + {{ return('RELY') }} + {%- endif -%} + {%- endfor -%} + {{ return('') }} +{%- endmacro -%} {#- This macro is called internally and passed which constraint types to create. -#} {%- macro create_constraints_by_type(constraint_types, quote_columns, lookup_cache) -%} - {#- Loop through the results and find all tests that passed and match the constraint_types -#} - {#- Issue #2: added condition that the where config must be empty -#} - {%- for res in results - if res.node.config.materialized == "test" - and res.status in ("pass", "warn") - and res.node.test_metadata - and res.node.test_metadata.name is in( constraint_types ) - and ( res.failures == 0 or - res.node.config.get("always_create_constraint", false) ) - and ( res.node.config.where is none or - res.node.config.get("always_create_constraint", false) ) - and res.node.config.get("dbt_constraints_enabled", true) -%} - - {%- set test_model = res.node -%} + {#- Loop through the metadata and find all tests that match the constraint_types -#} + {%- for test_model in graph.nodes.values() | selectattr("resource_type", "equalto", "test") + if test_model.test_metadata + and test_model.test_metadata.name is in( constraint_types ) + and test_model.config.enabled + and test_model.config.get("dbt_constraints_enabled", true) -%} + {%- set test_parameters = test_model.test_metadata.kwargs -%} - {% set ns = namespace(verify_permissions=false) %} + {%- set rely_clause = '' -%} + {%- if test_model.config.where -%} + {#- Set NORELY if there is a condition on the test -#} + {%- set rely_clause = 'NORELY' -%} + {%- else -%} + {%- set rely_clause = lookup_should_rely(test_model.unique_id) -%} + {%- endif -%} + {%- set should_generate_constraint = true + if ( rely_clause != '' and not test_model.config.where ) + or test_model.config.get("always_create_constraint", false) + else false -%} - {#- Find the table models that are referenced by this test. - These models must be physical tables and cannot be sources -#} + {% set ns = namespace(verify_permissions=false) %} {%- set table_models = [] -%} - {%- for node in graph.nodes.values() | selectattr("unique_id", "in", test_model.depends_on.nodes) - if node.resource_type in ( ( "model", "snapshot", "seed") ) - if node.config.materialized in( ("table", "incremental", "snapshot", "seed") ) -%} - - {#- Append to our list of models &or snapshots for this test -#} - {%- do table_models.append(node) -%} - - {% endfor %} - {#- Check if we allow constraints on sources overall and for this specific type of constraint -#} - {%- if var('dbt_constraints_sources_enabled', false) and ( - ( var('dbt_constraints_sources_pk_enabled', false) and test_model.test_metadata.name in("primary_key") ) - or ( var('dbt_constraints_sources_uk_enabled', false) and test_model.test_metadata.name in("unique_key", "unique_combination_of_columns", "unique") ) - or ( var('dbt_constraints_sources_fk_enabled', false) and test_model.test_metadata.name in("foreign_key", "relationships") ) - or ( var('dbt_constraints_sources_nn_enabled', false) and test_model.test_metadata.name in("not_null") ) - ) -%} - {%- for node in graph.sources.values() - | selectattr("resource_type", "equalto", "source") - | selectattr("unique_id", "in", test_model.depends_on.nodes) -%} - - {%- do node.update({'alias': node.alias or node.name }) -%} - {#- Append to our list of models for this test -#} - {%- do table_models.append(node) -%} + {#- Find the table models that are referenced by this test. -#} + {%- for table_node in test_model.depends_on.nodes -%} + {%- for node in graph.nodes.values() | selectattr("unique_id", "equalto", table_node) + if node.config.get("materialized", "other") not in ("view", "ephemeral") + and ( node.resource_type in ("model", "snapshot", "seed") + or ( node.resource_type == "source" and var('dbt_constraints_sources_enabled', false) + and ( ( var('dbt_constraints_sources_pk_enabled', false) and test_model.test_metadata.name in("primary_key") ) + or ( var('dbt_constraints_sources_uk_enabled', false) and test_model.test_metadata.name in("unique_key", "unique_combination_of_columns", "unique") ) + or ( var('dbt_constraints_sources_fk_enabled', false) and test_model.test_metadata.name in("foreign_key", "relationships") ) + or ( var('dbt_constraints_sources_nn_enabled', false) and test_model.test_metadata.name in("not_null") ) ) + ) ) -%} + + {%- do node.update({'alias': node.alias or node.name }) -%} + {#- Append to our list of models for this test -#} + {%- do table_models.append(node) -%} + {%- if node.resource_type == "source" -%} {#- If we are using a sources, we will need to verify permissions -#} {%- set ns.verify_permissions = true -%} + {%- endif -%} - {%- endfor -%} - {%- endif -%} - + {% endfor %} + {% endfor %} {#- We only create PK/UK if there is one model referenced by the test and if all the columns exist as physical columns on the table -#} {%- if 1 == table_models|count - and test_model.test_metadata.name in("primary_key", "unique_key", "unique_combination_of_columns", "unique") -%} + and test_model.test_metadata.name in("primary_key", "unique_key", "unique_combination_of_columns", "unique") + and should_generate_constraint -%} {# Attempt to identify a parameter we can use for the column names #} {%- set column_names = [] -%} @@ -240,10 +252,10 @@ identifier=table_models[0].alias ) -%} {%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%} {%- if test_model.test_metadata.name == "primary_key" -%} - {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache) -%} - {%- do dbt_constraints.create_primary_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache) -%} + {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%} + {%- do dbt_constraints.create_primary_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} {%- else -%} - {%- do dbt_constraints.create_unique_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache) -%} + {%- do dbt_constraints.create_unique_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} {%- endif -%} {%- else -%} {%- do log("Skipping primary/unique key because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%} @@ -252,7 +264,8 @@ {#- We only create FK if there are two models referenced by the test and if all the columns exist as physical columns on the tables -#} {%- elif 2 == table_models|count - and test_model.test_metadata.name in( "foreign_key", "relationships") -%} + and test_model.test_metadata.name in( "foreign_key", "relationships") + and should_generate_constraint -%} {%- set fk_model = none -%} {%- set pk_model = none -%} @@ -317,7 +330,7 @@ {%- elif not dbt_constraints.table_columns_all_exist(fk_table_relation, fk_column_names, lookup_cache) -%} {%- do log("Skipping foreign key because a physical column was not found on the fk table: " ~ fk_model.name ~ " " ~ fk_column_names, info=true) -%} {%- else -%} - {%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache) -%} + {%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} {%- endif -%} {%- else -%} {%- do log("Skipping foreign key because a we couldn't find the child table: model=" ~ fk_model_names ~ " or source=" ~ fk_source_names, info=true) -%} @@ -326,7 +339,8 @@ {#- We only create NN if there is one model referenced by the test and if all the columns exist as physical columns on the table -#} {%- elif 1 == table_models|count - and test_model.test_metadata.name in("not_null") -%} + and test_model.test_metadata.name in("not_null") + and should_generate_constraint -%} {# Attempt to identify a parameter we can use for the column names #} {%- set column_names = [] -%} @@ -348,7 +362,7 @@ identifier=table_models[0].alias ) -%} {%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%} - {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache) -%} + {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- else -%} {%- do log("Skipping not null constraint because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%} {%- endif -%} From 5792c73fbd4fc7f837dbfa090cb658648c0e3d9a Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 15 Jul 2024 18:17:34 -0400 Subject: [PATCH 06/16] Added configuration to test always_create_constraint logic --- integration_tests/dbt_project.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index e484ac3..e2fdcdc 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -50,3 +50,10 @@ seeds: +quote_columns: false +post-hook: "{{ clone_table('source_') }}" #+full_refresh: false + +tests: + dbt_constraints_integration_tests: + +always_create_constraint: true + # These configuration settings disable running tests or just constraints by path + # +enabled: false + #+dbt_constraints_enabled: false From da5eab9ae6afb1023fa5fbe441a936876f44e3e0 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 15 Jul 2024 18:45:49 -0400 Subject: [PATCH 07/16] Updated docs for NORELY and always_create_constraint --- README.md | 52 ++++++++++++++++++---------------------------------- 1 file changed, 18 insertions(+), 34 deletions(-) diff --git a/README.md b/README.md index 66fc05e..23acb8f 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ vars: ```yml packages: - package: Snowflake-Labs/dbt_constraints - version: [">=0.6.0", "<0.7.0"] + version: [">=0.7.0", "<0.8.0"] # for the latest version tag. # You can also pull the latest changes from Github with the following: # - git: "https://github.com/Snowflake-Labs/dbt_constraints.git" @@ -119,17 +119,22 @@ packages: __have_ownership_priv(table_relation, verify_permissions, lookup_cache=none) ``` +## RELY and NORELY Properties + +Version 0.7.0 introduces the ability to create constraints for failed tests on Snowflake. On Snowflake, executed tests with zero failures are created with the `RELY` property. Failed tests will generate `NORELY` constraints and constraints will be altered to `RELY` or `NORELY` based on subsequent executions of the test. It is also possible to create `NORELY` constraints using `dbt run` and then have those constraints become RELY constraints using `dbt test`. + + ## dbt_constraints Limitations Generally, if you don't meet a requirement, tests are still executed but the constraint is skipped rather than producing an error. -* All models involved in a constraint must be materialized as table, incremental, snapshot, or seed. +* All models involved in a constraint must not be a view or ephemeral materialization. * If source constraints are enabled, the source must be a table. You must also have the `OWNERSHIP` table privilege to add a constraint. For foreign keys you also need the `REFERENCES` privilege on the parent table with the primary or unique key. The package will identify when you lack these privileges on Snowflake and PostgreSQL. Oracle does not provide an easy way to look up your effective privileges so it has an exception handler and will display Oracle's error messages. * All columns on constraints must be individual column names, not expressions. You can reference columns on a model that come from an expression. -* Constraints are not created for failed tests. See how to get around this using severity and `config: always_create_constraint: true` in the next section. +* Constraints are only created if you execute a test. See how to get around this using `always_create_constraint: true` in the next section. * `primary_key`, `unique_key`, and `foreign_key` tests are considered first and duplicate constraints are skipped. One exception is that you will get an error if you add two different `primary_key` tests to the same model. @@ -139,46 +144,25 @@ Generally, if you don't meet a requirement, tests are still executed but the con * The `foreign_key` test will ignore any rows with a null column, even if only one of two columns in a compound key is null. If you also want to ensure FK columns are not null, you should add standard `not_null` tests to your model which will add not null constraints to the table. -* Referential constraints must apply to all the rows in a table so any tests with a `config: where:` property will be skipped when creating constraints. See how to disable this rule using `config: always_create_constraint: true` in the next section. +* Referential constraints must apply to all the rows in a table so any tests with a `config: where:` property will be set as `NORELY` when creating constraints. -## Advanced: `config: always_create_constraint: true` property +* You may need to manually drop a primary key constraint from a table if you change the columns in the constraint. This is not necessary for table materializations or if you do a full-refresh of an incremental model. -There is an advanced option to force a constraint to be generated when there is a `config: where:` property or if the constraint has a threshold. The `config: always_create_constraint: true` property will override those exclusions. When this setting is in effect, you can create constraints even when you have excluded some records or have a number of failures below a threshold. If your test has a status of 'failed', it will still be skipped. Please see [dbt's documentation on how to set a threshold for failures](https://docs.getdbt.com/reference/resource-configs/severity). +## Advanced: `always_create_constraint: true` Property + +There is an advanced option to force a constraint to be generated even when the test was not executed. When this setting is in effect, constraints on Snowflake will have the `NORELY` property until the associated test is executed with zero failures. Snowflake does not support `NORELY` for not null constraints so those constraints will still be skipped. You activate this feature in your dbt_project.yml under the `tests:` section. You can set it to be true for your entire project or you can specify specific folders that should use this feature. __Caveat Emptor:__ * You will get an error if you try to force constraints to be generated that are enforced by your database. On Snowflake that is only a not_null constraint but on databases like Oracle, all the generated constraints are enforced. -* This feature could cause unexpected query results on Snowflake due to [join elimination](https://docs.snowflake.com/en/user-guide/join-elimination). +* This feature could cause unexpected query results on Snowflake due to [join elimination](https://docs.snowflake.com/en/user-guide/join-elimination). Although executing tests on Snowflake will correctly set the `RELY` or `NORELY` property based on whether the tests pass and fail, activating this feature and skipping the execution of tests will not cause a `RELY` constraint to become a `NORELY` constraint. A `RELY` constraint only becomes a `NORELY` constraint if a test is executed and has failures. If you create a `RELY` constraint by running `dbt build` and subsequently only execute `dbt run` without following up with `dbt test`, you could have constraints that still have the `RELY` property but now have referential integrity issues. Users are encouraged to frequently or always execute their tests so that the `RELY` property is kept up to date. -This is an example using the feature: +This is an example from a dbt_project.yml using the feature: ```yml - - name: dim_duplicate_orders - description: "Test that we do not try to create PK/UK on failed tests" - columns: - - name: o_orderkey - description: "The primary key for this table" - - name: o_orderkey_seq - description: "duplicate seq column to test UK" - tests: - # This constraint should be skipped because it has failures - - dbt_constraints.primary_key: - column_name: o_orderkey - config: - severity: warn - # This constraint should be still generated because always_create_constraint=true - - dbt_constraints.unique_key: - column_name: o_orderkey - config: - warn_if: ">= 5000" - error_if: ">= 10000" - always_create_constraint: true - # This constraint should be still generated because always_create_constraint=true - - dbt_constraints.unique_key: - column_name: o_orderkey_seq - config: - severity: warn - always_create_constraint: true +tests: + your_project_name: + +always_create_constraint: true ``` ## Primary Maintainers From 3c9421b96e2196007a8355c863a0334128cd518f Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Mon, 22 Jul 2024 13:41:31 -0400 Subject: [PATCH 08/16] Made always_create_constraint respect selected models --- macros/create_constraints.sql | 74 ++++++++++++++++++++---- macros/snowflake__create_constraints.sql | 12 ++++ 2 files changed, 76 insertions(+), 10 deletions(-) diff --git a/macros/create_constraints.sql b/macros/create_constraints.sql index cd2de28..0486fa8 100644 --- a/macros/create_constraints.sql +++ b/macros/create_constraints.sql @@ -160,11 +160,61 @@ {%- endmacro -%} +{#- This macro checks if a test or its model is selected -#} +{%- macro test_selected(test_model) -%} + + {%- if test_model.unique_id in selected_resources + or lists_intersect(test_model.depends_on.nodes, selected_resources) -%} + {{ return(true) }} + {%- endif -%} + + {#- Check if a PK/UK should be created because it is referenced by a selected FK -#} + {%- if test_model.test_metadata + and test_model.test_metadata.name in ("primary_key", "unique_key", "unique_combination_of_columns", "unique") -%} + {%- set pk_test_args = test_model.test_metadata.kwargs -%} + {%- set pk_test_columns = [] -%} + {%- if pk_test_args.column_names -%} + {%- set pk_test_columns = pk_test_args.column_names -%} + {%- elif pk_test_args.combination_of_columns -%} + {%- set pk_test_columns = pk_test_args.combination_of_columns -%} + {%- elif pk_test_args.column_name -%} + {%- set pk_test_columns = [pk_test_args.column_name] -%} + {%- endif -%} + {%- for fk_model in graph.nodes.values() | selectattr("resource_type", "equalto", "test") + if fk_model.test_metadata + and fk_model.test_metadata.name in ("foreign_key", "relationships") + and lists_intersect(test_model.depends_on.nodes, fk_model.depends_on.nodes) + and ( fk_model.unique_id in selected_resources + or lists_intersect(fk_model.depends_on.nodes, selected_resources) ) -%} + {%- set fk_test_args = fk_model.test_metadata.kwargs -%} + {%- set fk_test_columns = [] -%} + {%- if fk_test_args.pk_column_names -%} + {%- set fk_test_columns = fk_test_args.pk_column_names -%} + {%- elif fk_test_args.pk_column_name -%} + {%- set fk_test_columns = [fk_test_args.pk_column_name] -%} + {%- elif fk_test_args.field -%} + {%- set fk_test_columns = [fk_test_args.field] -%} + {%- endif -%} + {%- if column_list_matches(pk_test_columns, fk_test_columns) -%} + {{ return(true) }} + {%- endif -%} + {%- endfor -%} + {%- endif -%} + + {{ return(false) }} +{%- endmacro -%} + + {#- This macro that checks if a test has results and whether there were errors -#} -{%- macro lookup_should_rely(test_unique_id) -%} +{%- macro lookup_should_rely(test_model) -%} + {%- if test_model.config.where -%} + {#- Set NORELY if there is a condition on the test -#} + {%- set rely_clause = 'NORELY' -%} + {%- endif -%} + {%- for res in results if res.node.config.materialized == "test" - and res.node.unique_id == test_unique_id -%} + and res.node.unique_id == test_model.unique_id -%} {%- if res.failures > 0 -%} {#- Set NORELY if there is a test failure -#} {{ return('NORELY') }} @@ -188,16 +238,12 @@ and test_model.config.get("dbt_constraints_enabled", true) -%} {%- set test_parameters = test_model.test_metadata.kwargs -%} - {%- set rely_clause = '' -%} - {%- if test_model.config.where -%} - {#- Set NORELY if there is a condition on the test -#} - {%- set rely_clause = 'NORELY' -%} - {%- else -%} - {%- set rely_clause = lookup_should_rely(test_model.unique_id) -%} - {%- endif -%} + {%- set rely_clause = lookup_should_rely(test_model) -%} + {%- set should_generate_constraint = true if ( rely_clause != '' and not test_model.config.where ) - or test_model.config.get("always_create_constraint", false) + or (test_model.config.get("always_create_constraint", false) + and test_selected(test_model) ) else false -%} {% set ns = namespace(verify_permissions=false) %} @@ -422,3 +468,11 @@ {{ return(false) }} {%- endif -%} {%- endmacro -%} + +{# This macro allows us to compare two lists to see if they intersect #} +{%- macro lists_intersect(listA, listB) -%} + {%- for valueFromA in listA if valueFromA in listB -%} + {{ return(true) }} + {% endfor %} + {{ return(false) }} +{%- endmacro -%} diff --git a/macros/snowflake__create_constraints.sql b/macros/snowflake__create_constraints.sql index 93e7764..d3960b7 100644 --- a/macros/snowflake__create_constraints.sql +++ b/macros/snowflake__create_constraints.sql @@ -7,6 +7,10 @@ {%- set existing_constraint = dbt_constraints.unique_constraint_exists(table_relation, column_names, lookup_cache) -%} {%- if constraint_name == existing_constraint -%} {%- do set_rely_norely(table_relation, constraint_name, lookup_cache.unique_keys[table_relation][constraint_name].rely, rely_clause) -%} + {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: + { "constraint_name": constraint_name, + "columns": column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} {%- elif none == existing_constraint -%} {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} @@ -44,6 +48,10 @@ {%- set existing_constraint = dbt_constraints.unique_constraint_exists(table_relation, column_names, lookup_cache) -%} {%- if constraint_name == existing_constraint -%} {%- do set_rely_norely(table_relation, constraint_name, lookup_cache.unique_keys[table_relation][constraint_name].rely, rely_clause) -%} + {%- do lookup_cache.unique_keys.update({table_relation: {constraint_name: + { "constraint_name": constraint_name, + "columns": column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} {%- elif none == existing_constraint -%} {%- if dbt_constraints.have_ownership_priv(table_relation, verify_permissions, lookup_cache) -%} @@ -84,6 +92,10 @@ {%- set existing_constraint = dbt_constraints.foreign_key_exists(fk_table_relation, fk_column_names, lookup_cache) -%} {%- if constraint_name == existing_constraint -%} {%- do set_rely_norely(fk_table_relation, constraint_name, lookup_cache.foreign_keys[fk_table_relation][constraint_name].rely, rely_clause) -%} + {%- do lookup_cache.foreign_keys.update({fk_table_relation: {constraint_name: + {"constraint_name": constraint_name, + "columns": fk_column_names, + "rely": "true" if rely_clause == "RELY" else "false" } } }) -%} {%- elif none == existing_constraint -%} {%- if dbt_constraints.have_ownership_priv(fk_table_relation, verify_permissions, lookup_cache) and dbt_constraints.have_references_priv(pk_table_relation, verify_permissions, lookup_cache) -%} From 6507a775e0789fff7247695b8947f8e2fd1ff3dc Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Thu, 25 Jul 2024 12:57:33 -0400 Subject: [PATCH 09/16] Added tests for always_create_constraint set on models --- integration_tests/models/dim_part.sql | 2 ++ integration_tests/models/schema.yml | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/integration_tests/models/dim_part.sql b/integration_tests/models/dim_part.sql index 2905fe0..a28693c 100644 --- a/integration_tests/models/dim_part.sql +++ b/integration_tests/models/dim_part.sql @@ -2,6 +2,8 @@ All Parts Additional unique keys generated by sequence and hash */ +{{ config(always_create_constraint = true) }} + SELECT P.*, DENSE_RANK() over (order by p_partkey) as p_partkey_seq diff --git a/integration_tests/models/schema.yml b/integration_tests/models/schema.yml index 8cf264c..299e632 100644 --- a/integration_tests/models/schema.yml +++ b/integration_tests/models/schema.yml @@ -200,6 +200,8 @@ models: - name: dim_orders_null_keys description: "All Orders" + config: + always_create_constraint: true columns: - name: o_custkey tests: @@ -213,10 +215,12 @@ models: column_name: o_orderkey config: severity: warn + # test that we still create this valid unique key - dbt_constraints.unique_key: column_name: o_orderkey_seq + - name: dim_part_supplier description: "Multi column UK" columns: From 276d178685829e739b85112ff5871f03e1107989 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Thu, 25 Jul 2024 12:58:22 -0400 Subject: [PATCH 10/16] ignore .gitconfig --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 05dada1..d470565 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ target/ dbt_packages/ logs/ .DS_Store +.gitconfig From 50e6cef119e1b4509570416a2689f6d454a9926a Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Thu, 25 Jul 2024 12:58:47 -0400 Subject: [PATCH 11/16] Update to version 1.0.0 --- dbt_project.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt_project.yml b/dbt_project.yml index 6055048..a32fb9f 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ name: 'dbt_constraints' -version: '0.6.3' +version: '1.0.0' config-version: 2 # These macros depend on the results and graph objects in dbt >=0.19.0 From 77b3a69b8e1ea78691925c057fd07fe6b066db69 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Thu, 25 Jul 2024 13:00:02 -0400 Subject: [PATCH 12/16] by default test always_create_constraint only on specific models/tests --- integration_tests/dbt_project.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index e2fdcdc..b18c802 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -52,8 +52,8 @@ seeds: #+full_refresh: false tests: - dbt_constraints_integration_tests: - +always_create_constraint: true + +dbt_constraints_integration_tests: + #+always_create_constraint: true # These configuration settings disable running tests or just constraints by path # +enabled: false #+dbt_constraints_enabled: false From cfb8108f35ef50a671a1cca22e04db79ceec168e Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Thu, 25 Jul 2024 13:00:57 -0400 Subject: [PATCH 13/16] Fixes for always_create_constraint, constraint selection --- macros/create_constraints.sql | 372 +++++++++++++---------- macros/snowflake__create_constraints.sql | 18 +- 2 files changed, 227 insertions(+), 163 deletions(-) diff --git a/macros/create_constraints.sql b/macros/create_constraints.sql index 0486fa8..1ac36dd 100644 --- a/macros/create_constraints.sql +++ b/macros/create_constraints.sql @@ -92,6 +92,17 @@ {%- endmacro -%} +{#- Define macro for whether a DB implementation has implemented logic for RELY and NORELY constraints -#} + +{%- macro adapter_supports_rely_norely(test_name) -%} + {{ return(adapter.dispatch('adapter_supports_rely_norely', 'dbt_constraints')(test_name)) }} +{%- endmacro -%} + +{#- By default, we assume DB implementations have NOT implemented logic for RELY and NORELY constraints -#} +{%- macro default__adapter_supports_rely_norely(test_name) -%} + {{ return(false) }} +{%- endmacro -%} + {#- Override dbt's truncate_relation macro to allow us to create adapter specific versions that drop constraints -#} @@ -163,9 +174,11 @@ {#- This macro checks if a test or its model is selected -#} {%- macro test_selected(test_model) -%} - {%- if test_model.unique_id in selected_resources - or lists_intersect(test_model.depends_on.nodes, selected_resources) -%} - {{ return(true) }} + {%- if test_model.unique_id in selected_resources -%} + {{ return("TEST_SELECTED") }} + {%- endif -%} + {%- if lists_intersect(test_model.depends_on.nodes, selected_resources) -%} + {{ return("MODEL_SELECTED") }} {%- endif -%} {#- Check if a PK/UK should be created because it is referenced by a selected FK -#} @@ -196,12 +209,12 @@ {%- set fk_test_columns = [fk_test_args.field] -%} {%- endif -%} {%- if column_list_matches(pk_test_columns, fk_test_columns) -%} - {{ return(true) }} + {{ return("PK_UK_FOR_SELECTED_FK") }} {%- endif -%} {%- endfor -%} {%- endif -%} - {{ return(false) }} + {{ return(none) }} {%- endmacro -%} @@ -227,6 +240,22 @@ {%- endmacro -%} +{#- This macro that checks if a test or its model has always_create_constraint set -#} +{%- macro should_always_create_constraint(test_model) -%} + {%- if test_model.config.get("always_create_constraint", false) == true -%} + {{ return(true) }} + {%- endif -%} + {%- for table_node in test_model.depends_on.nodes -%} + {%- for node in graph.nodes.values() | selectattr("unique_id", "equalto", table_node) + if node.config.get("always_create_constraint", false) == true -%} + {{ return(true) }} + {%- endfor -%} + {%- endfor -%} + + {{ return(false) }} +{%- endmacro -%} + + {#- This macro is called internally and passed which constraint types to create. -#} {%- macro create_constraints_by_type(constraint_types, quote_columns, lookup_cache) -%} @@ -238,183 +267,202 @@ and test_model.config.get("dbt_constraints_enabled", true) -%} {%- set test_parameters = test_model.test_metadata.kwargs -%} - {%- set rely_clause = lookup_should_rely(test_model) -%} - - {%- set should_generate_constraint = true - if ( rely_clause != '' and not test_model.config.where ) - or (test_model.config.get("always_create_constraint", false) - and test_selected(test_model) ) - else false -%} - - {% set ns = namespace(verify_permissions=false) %} - {%- set table_models = [] -%} - - {#- Find the table models that are referenced by this test. -#} - {%- for table_node in test_model.depends_on.nodes -%} - {%- for node in graph.nodes.values() | selectattr("unique_id", "equalto", table_node) - if node.config.get("materialized", "other") not in ("view", "ephemeral") - and ( node.resource_type in ("model", "snapshot", "seed") - or ( node.resource_type == "source" and var('dbt_constraints_sources_enabled', false) - and ( ( var('dbt_constraints_sources_pk_enabled', false) and test_model.test_metadata.name in("primary_key") ) - or ( var('dbt_constraints_sources_uk_enabled', false) and test_model.test_metadata.name in("unique_key", "unique_combination_of_columns", "unique") ) - or ( var('dbt_constraints_sources_fk_enabled', false) and test_model.test_metadata.name in("foreign_key", "relationships") ) - or ( var('dbt_constraints_sources_nn_enabled', false) and test_model.test_metadata.name in("not_null") ) ) - ) ) -%} - - {%- do node.update({'alias': node.alias or node.name }) -%} - {#- Append to our list of models for this test -#} - {%- do table_models.append(node) -%} - {%- if node.resource_type == "source" -%} - {#- If we are using a sources, we will need to verify permissions -#} - {%- set ns.verify_permissions = true -%} - {%- endif -%} + {%- set test_name = test_model.test_metadata.name -%} + {%- set selected = test_selected(test_model) -%} + + {#- We can shortcut additional tests if the constraint was not selected -#} + {%- if selected is not none -%} + {#- rely_clause clause will be RELY if a test passed, NORELY if it failed, and '' if it was skipped -#} + {%- set rely_clause = lookup_should_rely(test_model) -%} + {%- set always_create_constraint = should_always_create_constraint(test_model) -%} + {%- else -%} + {%- set rely_clause = '' -%} + {%- set always_create_constraint = false -%} + {%- endif -%} + {#- Create constraints that: + - Either the test or its model was selected to run, including PK/UK for FK + - Passed the test (RELY) or the database supports NORELY constraints + - We ran the test (RELY/NORELY) or we need the constraint for a FK + or we have the always_create_constraint parameter turned on -#} + {%- if selected is not none + and ( rely_clause == 'RELY' + or dbt_constraints.adapter_supports_rely_norely(test_name) == true ) + and ( rely_clause in('RELY', 'NORELY') + or selected == "PK_UK_FOR_SELECTED_FK" + or always_create_constraint == true ) -%} + + {% set ns = namespace(verify_permissions=false) %} + {%- set table_models = [] -%} + + {#- Find the table models that are referenced by this test. -#} + {%- for table_node in test_model.depends_on.nodes -%} + {%- for node in graph.nodes.values() | selectattr("unique_id", "equalto", table_node) + if node.config.get("materialized", "other") not in ("view", "ephemeral") + and ( node.resource_type in ("model", "snapshot", "seed") + or ( node.resource_type == "source" and var('dbt_constraints_sources_enabled', false) + and ( ( var('dbt_constraints_sources_pk_enabled', false) and test_name in("primary_key") ) + or ( var('dbt_constraints_sources_uk_enabled', false) and test_name in("unique_key", "unique_combination_of_columns", "unique") ) + or ( var('dbt_constraints_sources_fk_enabled', false) and test_name in("foreign_key", "relationships") ) + or ( var('dbt_constraints_sources_nn_enabled', false) and test_name in("not_null") ) ) + ) ) -%} + + {%- do node.update({'alias': node.alias or node.name }) -%} + {#- Append to our list of models for this test -#} + {%- do table_models.append(node) -%} + {%- if node.resource_type == "source" + or node.config.get("materialized", "other") not in ("table", "incremental", "snapshot", "seed") -%} + {#- If we are using a sources or custom materializations, we will need to verify permissions -#} + {%- set ns.verify_permissions = true -%} + {%- endif -%} + + {% endfor %} {% endfor %} - {% endfor %} - {#- We only create PK/UK if there is one model referenced by the test - and if all the columns exist as physical columns on the table -#} - {%- if 1 == table_models|count - and test_model.test_metadata.name in("primary_key", "unique_key", "unique_combination_of_columns", "unique") - and should_generate_constraint -%} - - {# Attempt to identify a parameter we can use for the column names #} - {%- set column_names = [] -%} - {%- if test_parameters.column_names -%} - {%- set column_names = test_parameters.column_names -%} - {%- elif test_parameters.combination_of_columns -%} - {%- set column_names = test_parameters.combination_of_columns -%} - {%- elif test_parameters.column_name -%} - {%- set column_names = [test_parameters.column_name] -%} - {%- else -%} - {{ exceptions.raise_compiler_error( - "`column_names` or `column_name` parameter missing for primary/unique key constraint on table: '" ~ table_models[0].name - ) }} - {%- endif -%} - - {%- set table_relation = api.Relation.create( - database=table_models[0].database, - schema=table_models[0].schema, - identifier=table_models[0].alias ) -%} - {%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%} - {%- if test_model.test_metadata.name == "primary_key" -%} - {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%} - {%- do dbt_constraints.create_primary_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} + {#- We only create PK/UK if there is one model referenced by the test + and if all the columns exist as physical columns on the table -#} + {%- if 1 == table_models|count + and test_name in("primary_key", "unique_key", "unique_combination_of_columns", "unique") -%} + + {# Attempt to identify a parameter we can use for the column names #} + {%- set column_names = [] -%} + {%- if test_parameters.column_names -%} + {%- set column_names = test_parameters.column_names -%} + {%- elif test_parameters.combination_of_columns -%} + {%- set column_names = test_parameters.combination_of_columns -%} + {%- elif test_parameters.column_name -%} + {%- set column_names = [test_parameters.column_name] -%} {%- else -%} - {%- do dbt_constraints.create_unique_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} + {{ exceptions.raise_compiler_error( + "`column_names` or `column_name` parameter missing for primary/unique key constraint on table: '" ~ table_models[0].name + ) }} {%- endif -%} - {%- else -%} - {%- do log("Skipping primary/unique key because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%} - {%- endif -%} - {#- We only create FK if there are two models referenced by the test - and if all the columns exist as physical columns on the tables -#} - {%- elif 2 == table_models|count - and test_model.test_metadata.name in( "foreign_key", "relationships") - and should_generate_constraint -%} - - {%- set fk_model = none -%} - {%- set pk_model = none -%} - {%- set fk_model_names = modules.re.findall( "(models|snapshots|seeds)\W+(\w+)" , test_model.file_key_name) -%} - {%- set fk_source_names = modules.re.findall( "source\W+(\w+)\W+(\w+)" , test_parameters.model) -%} - - {%- if 1 == fk_model_names | count -%} - {%- set fk_model = table_models | selectattr("name", "equalto", fk_model_names[0][1]) | first -%} - {%- set pk_model = table_models | rejectattr("name", "equalto", fk_model_names[0][1]) | first -%} - {%- elif 1 == fk_source_names | count -%} - {%- if table_models[0].source_name == fk_source_names[0][0] and table_models[0].name == fk_source_names[0][1] -%} - {%- set fk_model = table_models[0] -%} - {%- set pk_model = table_models[1] -%} + {%- set table_relation = api.Relation.create( + database=table_models[0].database, + schema=table_models[0].schema, + identifier=table_models[0].alias ) -%} + {%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%} + {%- if test_name == "primary_key" -%} + {%- if dbt_constraints.adapter_supports_rely_norely("not_null") == true -%} + {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%} + {%- endif -%} + {%- do dbt_constraints.create_primary_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} + {%- else -%} + {%- do dbt_constraints.create_unique_key(table_relation, column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} + {%- endif -%} {%- else -%} - {%- set fk_model = table_models[1] -%} - {%- set pk_model = table_models[0] -%} + {%- do log("Skipping primary/unique key because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%} {%- endif -%} - {%- endif -%} - {# {%- set fk_model_name = test_model.file_key_name |replace("models.", "") -%} #} - - {%- if fk_model and pk_model -%} - - {%- set fk_table_relation = api.Relation.create( - database=fk_model.database, - schema=fk_model.schema, - identifier=fk_model.alias) -%} - - {%- set pk_table_relation = api.Relation.create( - database=pk_model.database, - schema=pk_model.schema, - identifier=pk_model.alias) -%} - - {# Attempt to identify parameters we can use for the column names #} - {%- set pk_column_names = [] -%} - {%- if test_parameters.pk_column_names -%} - {%- set pk_column_names = test_parameters.pk_column_names -%} - {%- elif test_parameters.field -%} - {%- set pk_column_names = [test_parameters.field] -%} - {%- elif test_parameters.pk_column_name -%} - {%- set pk_column_names = [test_parameters.pk_column_name] -%} - {%- else -%} - {{ exceptions.raise_compiler_error( - "`pk_column_names`, `pk_column_name`, or `field` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters - ) }} + + {#- We only create FK if there are two models referenced by the test + and if all the columns exist as physical columns on the tables -#} + {%- elif 2 == table_models|count + and test_name in( "foreign_key", "relationships") -%} + + {%- set fk_model = none -%} + {%- set pk_model = none -%} + {%- set fk_model_names = modules.re.findall( "(models|snapshots|seeds)\W+(\w+)" , test_model.file_key_name) -%} + {%- set fk_source_names = modules.re.findall( "source\W+(\w+)\W+(\w+)" , test_parameters.model) -%} + + {%- if 1 == fk_model_names | count -%} + {%- set fk_model = table_models | selectattr("name", "equalto", fk_model_names[0][1]) | first -%} + {%- set pk_model = table_models | rejectattr("name", "equalto", fk_model_names[0][1]) | first -%} + {%- elif 1 == fk_source_names | count -%} + {%- if table_models[0].source_name == fk_source_names[0][0] and table_models[0].name == fk_source_names[0][1] -%} + {%- set fk_model = table_models[0] -%} + {%- set pk_model = table_models[1] -%} + {%- else -%} + {%- set fk_model = table_models[1] -%} + {%- set pk_model = table_models[0] -%} + {%- endif -%} + {%- endif -%} + {# {%- set fk_model_name = test_model.file_key_name |replace("models.", "") -%} #} + + {%- if fk_model and pk_model -%} + + {%- set fk_table_relation = api.Relation.create( + database=fk_model.database, + schema=fk_model.schema, + identifier=fk_model.alias) -%} + + {%- set pk_table_relation = api.Relation.create( + database=pk_model.database, + schema=pk_model.schema, + identifier=pk_model.alias) -%} + + {# Attempt to identify parameters we can use for the column names #} + {%- set pk_column_names = [] -%} + {%- if test_parameters.pk_column_names -%} + {%- set pk_column_names = test_parameters.pk_column_names -%} + {%- elif test_parameters.field -%} + {%- set pk_column_names = [test_parameters.field] -%} + {%- elif test_parameters.pk_column_name -%} + {%- set pk_column_names = [test_parameters.pk_column_name] -%} + {%- else -%} + {{ exceptions.raise_compiler_error( + "`pk_column_names`, `pk_column_name`, or `field` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters + ) }} + {%- endif -%} + + {%- set fk_column_names = [] -%} + {%- if test_parameters.fk_column_names -%} + {%- set fk_column_names = test_parameters.fk_column_names -%} + {%- elif test_parameters.column_name -%} + {%- set fk_column_names = [test_parameters.column_name] -%} + {%- elif test_parameters.fk_column_name -%} + {%- set fk_column_names = [test_parameters.fk_column_name] -%} + {%- else -%} + {{ exceptions.raise_compiler_error( + "`fk_column_names`, `fk_column_name`, or `column_name` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters + ) }} + {%- endif -%} + + {%- if not dbt_constraints.table_columns_all_exist(pk_table_relation, pk_column_names, lookup_cache) -%} + {%- do log("Skipping foreign key because a physical column was not found on the pk table: " ~ pk_model.name ~ " " ~ pk_column_names, info=true) -%} + {%- elif not dbt_constraints.table_columns_all_exist(fk_table_relation, fk_column_names, lookup_cache) -%} + {%- do log("Skipping foreign key because a physical column was not found on the fk table: " ~ fk_model.name ~ " " ~ fk_column_names, info=true) -%} + {%- else -%} + {%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} + {%- endif -%} + {%- else -%} + {%- do log("Skipping foreign key because a we couldn't find the child table: model=" ~ fk_model_names ~ " or source=" ~ fk_source_names, info=true) -%} {%- endif -%} - {%- set fk_column_names = [] -%} - {%- if test_parameters.fk_column_names -%} - {%- set fk_column_names = test_parameters.fk_column_names -%} - {%- elif test_parameters.column_name -%} - {%- set fk_column_names = [test_parameters.column_name] -%} - {%- elif test_parameters.fk_column_name -%} - {%- set fk_column_names = [test_parameters.fk_column_name] -%} - {%- else -%} + {#- We only create NN if there is one model referenced by the test + and if all the columns exist as physical columns on the table -#} + {%- elif 1 == table_models|count + and test_name in("not_null") -%} + + {# Attempt to identify a parameter we can use for the column names #} + {%- set column_names = [] -%} + {%- if test_parameters.column_names -%} + {%- set column_names = test_parameters.column_names -%} + {%- elif test_parameters.combination_of_columns -%} + {%- set column_names = test_parameters.combination_of_columns -%} + {%- elif test_parameters.column_name -%} + {%- set column_names = [test_parameters.column_name] -%} + {%- else -%} {{ exceptions.raise_compiler_error( - "`fk_column_names`, `fk_column_name`, or `column_name` parameter missing for foreign key constraint on table: '" ~ fk_model.name ~ " " ~ test_parameters + "`column_names` or `column_name` parameter missing for not null constraint on table: '" ~ table_models[0].name ) }} {%- endif -%} - {%- if not dbt_constraints.table_columns_all_exist(pk_table_relation, pk_column_names, lookup_cache) -%} - {%- do log("Skipping foreign key because a physical column was not found on the pk table: " ~ pk_model.name ~ " " ~ pk_column_names, info=true) -%} - {%- elif not dbt_constraints.table_columns_all_exist(fk_table_relation, fk_column_names, lookup_cache) -%} - {%- do log("Skipping foreign key because a physical column was not found on the fk table: " ~ fk_model.name ~ " " ~ fk_column_names, info=true) -%} + {%- set table_relation = api.Relation.create( + database=table_models[0].database, + schema=table_models[0].schema, + identifier=table_models[0].alias ) -%} + + {%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%} + {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- else -%} - {%- do dbt_constraints.create_foreign_key(pk_table_relation, pk_column_names, fk_table_relation, fk_column_names, ns.verify_permissions, quote_columns, test_parameters.constraint_name, lookup_cache, rely_clause) -%} + {%- do log("Skipping not null constraint because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%} {%- endif -%} - {%- else -%} - {%- do log("Skipping foreign key because a we couldn't find the child table: model=" ~ fk_model_names ~ " or source=" ~ fk_source_names, info=true) -%} - {%- endif -%} - {#- We only create NN if there is one model referenced by the test - and if all the columns exist as physical columns on the table -#} - {%- elif 1 == table_models|count - and test_model.test_metadata.name in("not_null") - and should_generate_constraint -%} - - {# Attempt to identify a parameter we can use for the column names #} - {%- set column_names = [] -%} - {%- if test_parameters.column_names -%} - {%- set column_names = test_parameters.column_names -%} - {%- elif test_parameters.combination_of_columns -%} - {%- set column_names = test_parameters.combination_of_columns -%} - {%- elif test_parameters.column_name -%} - {%- set column_names = [test_parameters.column_name] -%} - {%- else -%} - {{ exceptions.raise_compiler_error( - "`column_names` or `column_name` parameter missing for not null constraint on table: '" ~ table_models[0].name - ) }} {%- endif -%} - - {%- set table_relation = api.Relation.create( - database=table_models[0].database, - schema=table_models[0].schema, - identifier=table_models[0].alias ) -%} - - {%- if dbt_constraints.table_columns_all_exist(table_relation, column_names, lookup_cache) -%} - {%- do dbt_constraints.create_not_null(table_relation, column_names, ns.verify_permissions, quote_columns, lookup_cache, rely_clause) -%} - {%- else -%} - {%- do log("Skipping not null constraint because a physical column name was not found on the table: " ~ table_models[0].name ~ " " ~ column_names, info=true) -%} - {%- endif -%} - {%- endif -%} + {%- endfor -%} {%- endmacro -%} diff --git a/macros/snowflake__create_constraints.sql b/macros/snowflake__create_constraints.sql index d3960b7..35ed439 100644 --- a/macros/snowflake__create_constraints.sql +++ b/macros/snowflake__create_constraints.sql @@ -1,3 +1,19 @@ +{#- Snowflake supports RELY and NORELY constraints for PK, UK, FK but not not_null -#} +{%- macro snowflake__adapter_supports_rely_norely(test_name) -%} + {%- if test_name in ( + 'primary_key', + 'unique_key', + 'unique_combination_of_columns', + 'unique', + 'foreign_key', + 'relationships') -%} + {{ return(true) }} + {%- else -%} + {{ return(false) }} + {%- endif -%} +{%- endmacro -%} + + {# Snowflake specific implementation to create a primary key #} {%- macro snowflake__create_primary_key(table_relation, column_names, verify_permissions, quote_columns, constraint_name, lookup_cache, rely_clause) -%} {%- set constraint_name = (constraint_name or table_relation.identifier ~ "_" ~ column_names|join('_') ~ "_PK") | upper -%} @@ -130,7 +146,7 @@ {# Snowflake specific implementation to create a not null constraint #} {%- macro snowflake__create_not_null(table_relation, column_names, verify_permissions, quote_columns, lookup_cache, rely_clause) -%} {%- if not rely_clause == 'RELY' -%} - {%- do log("Skipping not null constraint for " ~ column_names | join(", ") ~ " in " ~ table_relation ~ " because Snowflake does not support NORELY for not null constraints.", info=false) -%} + {%- do log("Skipping not null constraint for " ~ column_names | join(", ") ~ " in " ~ table_relation ~ " because Snowflake does not support NORELY for not null constraints.", info=true) -%} {{ return(false) }} {%- endif -%} From 89df7497711954cf5e18e34248b13df332f93289 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Thu, 25 Jul 2024 13:01:58 -0400 Subject: [PATCH 14/16] Additional doc changes for v1 for NORELY and always_create_constraint --- README.md | 56 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 23acb8f..cc113b0 100644 --- a/README.md +++ b/README.md @@ -121,50 +121,64 @@ packages: ## RELY and NORELY Properties -Version 0.7.0 introduces the ability to create constraints for failed tests on Snowflake. On Snowflake, executed tests with zero failures are created with the `RELY` property. Failed tests will generate `NORELY` constraints and constraints will be altered to `RELY` or `NORELY` based on subsequent executions of the test. It is also possible to create `NORELY` constraints using `dbt run` and then have those constraints become RELY constraints using `dbt test`. +Version 1.0.0 introduces the ability to create constraints with the RELY and NORELY properties on Snowflake. Executed tests with zero failures are created with the `RELY` property. Tests with any failures will generate `NORELY` constraints and constraints will be altered to `RELY` or `NORELY` based on subsequent executions of the test. When the `always_create_constraint` feature is enabled, it is now also possible to create `NORELY` constraints using `dbt run` and then have those constraints become RELY constraints using `dbt test`. +## Determining the Constraints to Generate -## dbt_constraints Limitations - -Generally, if you don't meet a requirement, tests are still executed but the constraint is skipped rather than producing an error. - -* All models involved in a constraint must not be a view or ephemeral materialization. +Version 1.0.0 introduces a more advanced set of criteria for selecting tests to turn into constraints. +* The test must be one of the following: `primary_key`, `unique_key`, `unique_combination_of_columns`, `unique`, `foreign_key`, `relationships`, or `not_null` +* The test executed and had zero failures (RELY) or the database has support for NORELY constraints +* The test executed (RELY/NORELY), we need the primary/unique key constraint for a foreign key, or we have the `always_create_constraint` parameter turned on. +* If you are using `build`, `run`, or `test` for only part of a project using the `--select` parameter, either the test or its model was selected to run, or the test is a primary/unique key that is needed for a selected foreign key. If a primary/unique key is created for a foreign key, and its test was not executed, the primary/unique key will be created as a NORELY constraint. +* All models involved in a constraint must **not** be a view or ephemeral materialization. Version 1.0.0 now allows custom materializations. * If source constraints are enabled, the source must be a table. You must also have the `OWNERSHIP` table privilege to add a constraint. For foreign keys you also need the `REFERENCES` privilege on the parent table with the primary or unique key. The package will identify when you lack these privileges on Snowflake and PostgreSQL. Oracle does not provide an easy way to look up your effective privileges so it has an exception handler and will display Oracle's error messages. - * All columns on constraints must be individual column names, not expressions. You can reference columns on a model that come from an expression. - -* Constraints are only created if you execute a test. See how to get around this using `always_create_constraint: true` in the next section. - * `primary_key`, `unique_key`, and `foreign_key` tests are considered first and duplicate constraints are skipped. One exception is that you will get an error if you add two different `primary_key` tests to the same model. - * Foreign keys require that the parent table have a primary key or unique key on the referenced columns. Unique keys generated from standard `unique` tests are sufficient. - * The order of columns on a foreign key test must match between the FK columns and PK columns - -* The `foreign_key` test will ignore any rows with a null column, even if only one of two columns in a compound key is null. If you also want to ensure FK columns are not null, you should add standard `not_null` tests to your model which will add not null constraints to the table. - * Referential constraints must apply to all the rows in a table so any tests with a `config: where:` property will be set as `NORELY` when creating constraints. +Additional notes: +* The `foreign_key` test will ignore any rows with a null column, even if only one of two columns in a compound key is null. If you also want to ensure FK columns are not null, you should add standard `not_null` tests to your model which will add not null constraints to the table. * You may need to manually drop a primary key constraint from a table if you change the columns in the constraint. This is not necessary for table materializations or if you do a full-refresh of an incremental model. -## Advanced: `always_create_constraint: true` Property +## Advanced: `always_create_constraint` Property -There is an advanced option to force a constraint to be generated even when the test was not executed. When this setting is in effect, constraints on Snowflake will have the `NORELY` property until the associated test is executed with zero failures. Snowflake does not support `NORELY` for not null constraints so those constraints will still be skipped. You activate this feature in your dbt_project.yml under the `tests:` section. You can set it to be true for your entire project or you can specify specific folders that should use this feature. +There is an advanced option for Snowflake users to force a constraint to be generated even when the test was not executed. When this setting is in effect, constraints on Snowflake will have the `NORELY` property until the associated test is executed with zero failures. Snowflake does not support `NORELY` for not null constraints so those constraints will still be skipped. You activate this feature in your dbt_project.yml under the `models:` or `tests:` sections. You can set it to be true for your entire project or you can specify specific folders that should use this feature. You can also set this in a specific model's header. -__Caveat Emptor:__ +__[Caveat Emptor](https://en.wikipedia.org/wiki/Caveat_emptor):__ -* You will get an error if you try to force constraints to be generated that are enforced by your database. On Snowflake that is only a not_null constraint but on databases like Oracle, all the generated constraints are enforced. -* This feature could cause unexpected query results on Snowflake due to [join elimination](https://docs.snowflake.com/en/user-guide/join-elimination). Although executing tests on Snowflake will correctly set the `RELY` or `NORELY` property based on whether the tests pass and fail, activating this feature and skipping the execution of tests will not cause a `RELY` constraint to become a `NORELY` constraint. A `RELY` constraint only becomes a `NORELY` constraint if a test is executed and has failures. If you create a `RELY` constraint by running `dbt build` and subsequently only execute `dbt run` without following up with `dbt test`, you could have constraints that still have the `RELY` property but now have referential integrity issues. Users are encouraged to frequently or always execute their tests so that the `RELY` property is kept up to date. +* You will get an error if you try to force constraints to be generated that are enforced by your database. On Snowflake that is only a not_null constraint but on databases like Oracle, all the generated constraints are enforced. This is why, at present, only the Snowflake macros implement this feature. +* This feature can still cause unexpected query results on Snowflake due to [join elimination](https://docs.snowflake.com/en/user-guide/join-elimination). Although executing tests on Snowflake will correctly set the `RELY` or `NORELY` property based on whether the tests pass and fail, activating this feature and **skipping the execution of tests** will not cause a `RELY` constraint to become a `NORELY` constraint. A `RELY` constraint only becomes a `NORELY` constraint **if a test is executed** and has failures. If you create a `RELY` constraint by running `dbt build` and subsequently only execute `dbt run` without eventually following up with `dbt test`, you could have constraints that still have the `RELY` property but now have referential integrity issues. Snowflake users are encouraged to frequently or always execute their tests so that the `RELY` property is kept up to date. -This is an example from a dbt_project.yml using the feature: +These are examples from a dbt_project.yml using the feature in models or tests: ```yml +models: + your_project_name: + +always_create_constraint: true tests: your_project_name: +always_create_constraint: true ``` +This is an example from a model schema.yml using the feature. Setting the property in the `config:` section of a test does not work so you should set it in the model's `config:` section. + +```yml +version: 2 + +models: + - name: your_model_name + config: + always_create_constraint: true +``` + +This is an example of activating the feature in the header of a model: +```jinja +{{ config(always_create_constraint = true) }} +``` + ## Primary Maintainers * Dan Flippo ([@sfc-gh-dflippo](https://github.com/sfc-gh-dflippo)) From c821ac20f6e4fb997b16e62ce1ba61bd6043df49 Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Thu, 25 Jul 2024 16:29:51 -0400 Subject: [PATCH 15/16] Setting tests with warn_if and fail_calc to NORELY --- README.md | 2 +- macros/create_constraints.sql | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index cc113b0..e79c67b 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ Version 1.0.0 introduces a more advanced set of criteria for selecting tests to * `primary_key`, `unique_key`, and `foreign_key` tests are considered first and duplicate constraints are skipped. One exception is that you will get an error if you add two different `primary_key` tests to the same model. * Foreign keys require that the parent table have a primary key or unique key on the referenced columns. Unique keys generated from standard `unique` tests are sufficient. * The order of columns on a foreign key test must match between the FK columns and PK columns -* Referential constraints must apply to all the rows in a table so any tests with a `config: where:` property will be set as `NORELY` when creating constraints. +* Referential constraints must apply to all the rows in a table so any tests with a `config: where:`, `config: warn_if:`, or `config: fail_calc:` property will be set as `NORELY` when creating constraints. Additional notes: * The `foreign_key` test will ignore any rows with a null column, even if only one of two columns in a compound key is null. If you also want to ensure FK columns are not null, you should add standard `not_null` tests to your model which will add not null constraints to the table. diff --git a/macros/create_constraints.sql b/macros/create_constraints.sql index 1ac36dd..2b029ce 100644 --- a/macros/create_constraints.sql +++ b/macros/create_constraints.sql @@ -220,9 +220,11 @@ {#- This macro that checks if a test has results and whether there were errors -#} {%- macro lookup_should_rely(test_model) -%} - {%- if test_model.config.where -%} + {%- if test_model.config.where + or test_model.config.warn_if != "!= 0" + or test_model.config.fail_calc != "count(*)" -%} {#- Set NORELY if there is a condition on the test -#} - {%- set rely_clause = 'NORELY' -%} + {{ return('NORELY') }} {%- endif -%} {%- for res in results From c36ed0160fe63bc2c63aaede322d48df0c4ad5fc Mon Sep 17 00:00:00 2001 From: Dan Flippo Date: Thu, 25 Jul 2024 16:59:54 -0400 Subject: [PATCH 16/16] Updated version number in sample yaml for packages.yml --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e79c67b..a002b93 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ vars: ```yml packages: - package: Snowflake-Labs/dbt_constraints - version: [">=0.7.0", "<0.8.0"] + version: [">=1.0.0", "<1.1.0"] # for the latest version tag. # You can also pull the latest changes from Github with the following: # - git: "https://github.com/Snowflake-Labs/dbt_constraints.git"