From 2c75abfec9d57f63873ed1f65d3dfb41a38f2d7e Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Tue, 11 Jun 2024 14:42:01 -0700 Subject: [PATCH 01/10] Addressing issue 189 --- .../models/table/create_table_as.sql | 60 ++++++++++++++++--- .../materializations/models/table/table.sql | 36 ++--------- 2 files changed, 59 insertions(+), 37 deletions(-) diff --git a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql index 00c84bd..4c9fa84 100644 --- a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql @@ -1,12 +1,25 @@ {% macro fabric__create_table_as(temporary, relation, sql) -%} - {% set tmp_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} - {{ get_create_view_as_sql(tmp_relation, sql) }} + -- Make temp view relation + {% set tmp_vw_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + + -- drop temp relation if exists + {% do adapter.drop_relation(tmp_vw_relation) %} + + -- Fabric & Synapse adapters use temp relation because of lack of CTE support for CTE in CTAS, Insert + {{ get_create_view_as_sql(tmp_vw_relation, sql) }} + + -- Temp relation will come in use when contract is enforced or not + -- Making a temp relation + {% set temp_relation = make_temp_relation(relation, '__dbt_temp') %} + + -- Dropping a temp relation if it exists + {% do adapter.drop_relation(temp_relation) %} {% set contract_config = config.get('contract') %} {% if contract_config.enforced %} - CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}] + CREATE TABLE {{temp_relation}} {{ build_columns_constraints(relation) }} {{ get_assert_columns_equivalent(sql) }} {% set listColumns %} @@ -15,11 +28,44 @@ {% endfor %} {%endset%} - INSERT INTO [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}] - ({{listColumns}}) SELECT {{listColumns}} FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]; + INSERT INTO {{temp_relation}} ({{listColumns}}) + SELECT {{listColumns}} FROM {{tmp_vw_relation}}; {%- else %} - EXEC('CREATE TABLE [{{relation.database}}].[{{relation.schema}}].[{{relation.identifier}}] AS (SELECT * FROM [{{tmp_relation.database}}].[{{tmp_relation.schema}}].[{{tmp_relation.identifier}}]);'); + -- CTAS + {{ log("In CTAS") }} + {# EXEC('CREATE TABLE {{temp_relation}} AS SELECT * FROM {{tmp_vw_relation}} }}'); #} + SELECT 1 + EXEC('CREATE TABLE [{{temp_relation.database}}].[{{temp_relation.schema}}].[{{temp_relation.identifier}}] AS (SELECT * FROM [{{tmp_vw_relation.database}}].[{{tmp_vw_relation.schema}}].[{{tmp_vw_relation.identifier}}]);'); {% endif %} - {% do adapter.drop_relation(tmp_relation)%} + + -- making a backup relation, this will come in use when contract is enforced or not + {%- set backup_relation = make_backup_relation(relation, 'table') -%} + + -- Dropping a temp relation if it exists + {% do adapter.drop_relation(backup_relation) %} + + {%- set check_if_relation_exists = adapter.get_relation(database=relation.database, schema=relation.schema, identifier=relation.identifier) -%} + {% if check_if_relation_exists is not none %} + + {{ adapter.rename_relation(relation, backup_relation) }} + + -- Renaming temp relation as main relation + {{ adapter.rename_relation(temp_relation, relation) }} + + -- Drop backup relation + {% do adapter.drop_relation(backup_relation) %} + + {%- else %} + + -- Renaming temp relation as main relation + {{ adapter.rename_relation(temp_relation, relation) }} + + -- Dropping temp relation + {% do adapter.drop_relation(temp_relation) %} + + {% endif %} + + -- Dropping temp view relation + {% do adapter.drop_relation(tmp_vw_relation)%} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/models/table/table.sql b/dbt/include/fabric/macros/materializations/models/table/table.sql index 88f1df9..9d36a53 100644 --- a/dbt/include/fabric/macros/materializations/models/table/table.sql +++ b/dbt/include/fabric/macros/materializations/models/table/table.sql @@ -4,18 +4,11 @@ {%- set target_relation = this.incorporate(type='table') %} {%- set existing_relation = adapter.get_relation(database=this.database, schema=this.schema, identifier=this.identifier) -%} - {%- set backup_relation = none %} - {% if (existing_relation != none and existing_relation.type == "table") %} - {%- set backup_relation = make_backup_relation(target_relation, 'table') -%} - {% elif (existing_relation != none and existing_relation.type == "view") %} - {%- set backup_relation = make_backup_relation(target_relation, 'view') -%} - {% endif %} - - {% if (existing_relation != none) %} - -- drop the temp relations if they exist already in the database - {% do adapter.drop_relation(backup_relation) %} - -- Rename target relation as backup relation - {{ adapter.rename_relation(existing_relation, backup_relation) }} + {#-- Drop the relation if it was a view to "convert" it in a table. This may lead to + -- downtime, but it should be a relatively infrequent occurrence #} + {% if existing_relation is not none and not old_relation.is_table %} + {{ log("Dropping relation " ~ existing_relation ~ " because it is of type " ~ existing_relation.type) }} + {% do adapter.drop_relation(existing_relation) %} {% endif %} -- grab current tables grants config for comparision later on @@ -25,21 +18,11 @@ -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} - -- naming a temp relation - {% set tmp_relation = target_relation.incorporate(path={"identifier": target_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} - - -- Fabric & Synapse adapters use temp relation because of lack of CTE support for CTE in CTAS, Insert - -- drop temp relation if exists - {% do adapter.drop_relation(tmp_relation) %} - -- build model {% call statement('main') -%} - {{ get_create_table_as_sql(False, target_relation, sql) }} + {{ create_table_as(False, target_relation, sql) }} {%- endcall %} - -- drop temp relation if exists - {% do adapter.drop_relation(tmp_relation) %} - -- cleanup {{ run_hooks(post_hooks, inside_transaction=True) }} @@ -48,13 +31,6 @@ -- `COMMIT` happens here {{ adapter.commit() }} - {% if (backup_relation != none) %} - -- finally, drop the foreign key references if exists - {{ drop_fk_indexes_on_table(backup_relation) }} - -- drop existing/backup relation after the commit - {% do adapter.drop_relation(backup_relation) %} - {% endif %} - -- Add constraints including FK relation. {{ build_model_constraints(target_relation) }} {{ run_hooks(post_hooks, inside_transaction=False) }} From 3fca56e7978005aaaa7f38037409850ffde2390e Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Thu, 20 Jun 2024 13:47:02 -0700 Subject: [PATCH 02/10] Improving table materialization to minimize downtime --- dbt/adapters/fabric/__version__.py | 2 +- .../fabric/macros/adapters/relation.sql | 2 +- .../models/incremental/incremental.sql | 15 ++---- .../models/table/create_table_as.sql | 54 ++----------------- .../materializations/models/table/table.sql | 42 +++++++++++++-- tests/functional/adapter/test_constraints.py | 5 +- 6 files changed, 52 insertions(+), 68 deletions(-) diff --git a/dbt/adapters/fabric/__version__.py b/dbt/adapters/fabric/__version__.py index fd11d27..9675f25 100644 --- a/dbt/adapters/fabric/__version__.py +++ b/dbt/adapters/fabric/__version__.py @@ -1 +1 @@ -version = "1.8.6" +version = "1.8.7" diff --git a/dbt/include/fabric/macros/adapters/relation.sql b/dbt/include/fabric/macros/adapters/relation.sql index a6fe0f0..85aa0e2 100644 --- a/dbt/include/fabric/macros/adapters/relation.sql +++ b/dbt/include/fabric/macros/adapters/relation.sql @@ -1,4 +1,4 @@ -{% macro fabric__make_temp_relation(base_relation, suffix) %} +{% macro fabric__make_temp_relation(base_relation, suffix='__dbt_temp') %} {%- set temp_identifier = base_relation.identifier ~ suffix -%} {%- set temp_relation = base_relation.incorporate( path={"identifier": temp_identifier}) -%} diff --git a/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql b/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql index bd045df..4e8666b 100644 --- a/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql +++ b/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql @@ -3,7 +3,7 @@ {%- set full_refresh_mode = (should_full_refresh()) -%} {% set target_relation = this.incorporate(type='table') %} {%- set relation = load_cached_relation(this) -%} - + {%- set temp_relation = make_temp_relation(target_relation)-%} {%- set existing_relation = none %} {% if relation.type == 'table' %} {% set existing_relation = target_relation %} @@ -14,20 +14,12 @@ -- configs {%- set unique_key = config.get('unique_key') -%} {% set incremental_strategy = config.get('incremental_strategy') or 'default' %} - {%- set temp_relation = make_temp_relation(target_relation)-%} {% set grant_config = config.get('grants') %} {%- set on_schema_change = incremental_validate_on_schema_change(config.get('on_schema_change'), default='ignore') -%} {{ run_hooks(pre_hooks, inside_transaction=True) }} - -- naming a temp relation - {% set tmp_relation_view = target_relation.incorporate(path={"identifier": target_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} - - -- Fabric & Synapse adapters use temp relation because of lack of CTE support for CTE in CTAS, Insert - -- drop temp relation if exists - {% do adapter.drop_relation(tmp_relation_view) %} - {% if existing_relation is none %} {%- call statement('main') -%} {{ get_create_table_as_sql(False, target_relation, sql)}} @@ -36,7 +28,7 @@ {% elif existing_relation.is_view %} {#-- Can't overwrite a view with a table - we must drop --#} {{ log("Dropping relation " ~ target_relation ~ " because it is a view and this model is a table.") }} - {% do adapter.drop_relation(existing_relation) %} + {{ adapter.drop_relation(existing_relation) }} {%- call statement('main') -%} {{ get_create_table_as_sql(False, target_relation, sql)}} @@ -69,8 +61,7 @@ {%- endcall -%} {% endif %} - {% do adapter.drop_relation(tmp_relation_view) %} - {% do adapter.drop_relation(temp_relation) %} + {{ adapter.drop_relation(temp_relation) }} {{ run_hooks(post_hooks, inside_transaction=True) }} {% set target_relation = target_relation.incorporate(type='table') %} {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} diff --git a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql index 4c9fa84..8ed7ae2 100644 --- a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql @@ -1,25 +1,13 @@ {% macro fabric__create_table_as(temporary, relation, sql) -%} - -- Make temp view relation - {% set tmp_vw_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} - - -- drop temp relation if exists + {% set tmp_vw_relation = relation.incorporate(path={"identifier": relation.identifier ~ '_vw'}, type='view')-%} {% do adapter.drop_relation(tmp_vw_relation) %} - - -- Fabric & Synapse adapters use temp relation because of lack of CTE support for CTE in CTAS, Insert {{ get_create_view_as_sql(tmp_vw_relation, sql) }} - -- Temp relation will come in use when contract is enforced or not - -- Making a temp relation - {% set temp_relation = make_temp_relation(relation, '__dbt_temp') %} - - -- Dropping a temp relation if it exists - {% do adapter.drop_relation(temp_relation) %} - {% set contract_config = config.get('contract') %} {% if contract_config.enforced %} - CREATE TABLE {{temp_relation}} + CREATE TABLE {{relation}} {{ build_columns_constraints(relation) }} {{ get_assert_columns_equivalent(sql) }} {% set listColumns %} @@ -28,44 +16,12 @@ {% endfor %} {%endset%} - INSERT INTO {{temp_relation}} ({{listColumns}}) + INSERT INTO {{relation}} ({{listColumns}}) SELECT {{listColumns}} FROM {{tmp_vw_relation}}; {%- else %} - -- CTAS - {{ log("In CTAS") }} - {# EXEC('CREATE TABLE {{temp_relation}} AS SELECT * FROM {{tmp_vw_relation}} }}'); #} - SELECT 1 - EXEC('CREATE TABLE [{{temp_relation.database}}].[{{temp_relation.schema}}].[{{temp_relation.identifier}}] AS (SELECT * FROM [{{tmp_vw_relation.database}}].[{{tmp_vw_relation.schema}}].[{{tmp_vw_relation.identifier}}]);'); + EXEC('CREATE TABLE {{relation}} AS SELECT * FROM {{tmp_vw_relation}};'); {% endif %} - -- making a backup relation, this will come in use when contract is enforced or not - {%- set backup_relation = make_backup_relation(relation, 'table') -%} - - -- Dropping a temp relation if it exists - {% do adapter.drop_relation(backup_relation) %} - - {%- set check_if_relation_exists = adapter.get_relation(database=relation.database, schema=relation.schema, identifier=relation.identifier) -%} - {% if check_if_relation_exists is not none %} - - {{ adapter.rename_relation(relation, backup_relation) }} - - -- Renaming temp relation as main relation - {{ adapter.rename_relation(temp_relation, relation) }} - - -- Drop backup relation - {% do adapter.drop_relation(backup_relation) %} - - {%- else %} - - -- Renaming temp relation as main relation - {{ adapter.rename_relation(temp_relation, relation) }} - - -- Dropping temp relation - {% do adapter.drop_relation(temp_relation) %} - - {% endif %} - - -- Dropping temp view relation - {% do adapter.drop_relation(tmp_vw_relation)%} + {% do adapter.drop_relation(tmp_vw_relation) %} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/models/table/table.sql b/dbt/include/fabric/macros/materializations/models/table/table.sql index 9d36a53..3431761 100644 --- a/dbt/include/fabric/macros/materializations/models/table/table.sql +++ b/dbt/include/fabric/macros/materializations/models/table/table.sql @@ -6,22 +6,56 @@ {#-- Drop the relation if it was a view to "convert" it in a table. This may lead to -- downtime, but it should be a relatively infrequent occurrence #} - {% if existing_relation is not none and not old_relation.is_table %} + {% if existing_relation is not none and not existing_relation.is_table %} {{ log("Dropping relation " ~ existing_relation ~ " because it is of type " ~ existing_relation.type) }} - {% do adapter.drop_relation(existing_relation) %} + {{ adapter.drop_relation(existing_relation) }} {% endif %} -- grab current tables grants config for comparision later on {% set grant_config = config.get('grants') %} + -- Making a temp relation + {% set temp_relation = make_temp_relation(target_relation, '__dbt_temp') %} + + -- Drop temp relation if it exists before materializing temp relation + {{ adapter.drop_relation(temp_relation) }} + + {% set tmp_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '_vw'}, type='view')-%} + {{ run_hooks(pre_hooks, inside_transaction=False) }} -- `BEGIN` happens here: {{ run_hooks(pre_hooks, inside_transaction=True) }} -- build model {% call statement('main') -%} - {{ create_table_as(False, target_relation, sql) }} - {%- endcall %} + {{ create_table_as(False, temp_relation, sql) }} + {% endcall %} + + {% if existing_relation is not none and existing_relation.is_table %} + + -- making a backup relation, this will come in use when contract is enforced or not + {%- set backup_relation = make_backup_relation(existing_relation, 'table') -%} + + -- Dropping a temp relation if it exists + {{ adapter.drop_relation(backup_relation) }} + + -- Rename existing relation to back up relation + {{ adapter.rename_relation(existing_relation, backup_relation) }} + + -- Renaming temp relation as main relation + {{ adapter.rename_relation(temp_relation, target_relation) }} + + -- Drop backup relation + {{ adapter.drop_relation(backup_relation) }} + + {%- else %} + + -- Renaming temp relation as main relation + {{ adapter.rename_relation(temp_relation, target_relation) }} + + {% endif %} + + {{ adapter.drop_relation(tmp_vw_relation) }} -- cleanup {{ run_hooks(post_hooks, inside_transaction=True) }} diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index 6759424..a72a358 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -618,7 +618,10 @@ def test__constraints_enforcement_rollback( # Verify the previous table still exists relation = relation_from_name(project.adapter, "my_model") - model_backup = str(relation).replace("my_model", "my_model__dbt_backup") + # table materialization does not rename existing relation to back)up relation + # Rather, a new relation is created with __dbt_temp suffix. + # If model creation is successful, then the existing model is renamed as backup and then dropped + model_backup = str(relation).replace("my_model", "my_model") old_model_exists_sql = f"select * from {model_backup}" old_model_exists = project.run_sql(old_model_exists_sql, fetch="all") assert len(old_model_exists) == 1 From 640f4bed4b19dd602f2f23a7a81ba66e9f3193c7 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 21 Jun 2024 01:07:01 -0700 Subject: [PATCH 03/10] handling temp tables and views #188 --- .../models/incremental/incremental.sql | 32 +++++++++++-------- .../models/table/create_table_as.sql | 5 ++- .../materializations/models/table/table.sql | 4 ++- .../materializations/snapshots/helpers.sql | 8 +++++ .../materializations/snapshots/snapshot.sql | 6 ++-- 5 files changed, 34 insertions(+), 21 deletions(-) diff --git a/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql b/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql index 4e8666b..4599ba2 100644 --- a/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql +++ b/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql @@ -3,12 +3,14 @@ {%- set full_refresh_mode = (should_full_refresh()) -%} {% set target_relation = this.incorporate(type='table') %} {%- set relation = load_cached_relation(this) -%} - {%- set temp_relation = make_temp_relation(target_relation)-%} {%- set existing_relation = none %} {% if relation.type == 'table' %} {% set existing_relation = target_relation %} {% elif relation.type == 'view' %} {% set existing_relation = get_or_create_relation(relation.database, relation.schema, relation.identifier, relation.type)[1] %} + {#-- Can't overwrite a view with a table - we must drop --#} + {{ log("Dropping relation " ~ existing_relation ~ " because it is a view and target is a table.") }} + {{ adapter.drop_relation(existing_relation) }} {% endif %} -- configs @@ -20,29 +22,30 @@ {{ run_hooks(pre_hooks, inside_transaction=True) }} - {% if existing_relation is none %} - {%- call statement('main') -%} - {{ get_create_table_as_sql(False, target_relation, sql)}} - {%- endcall -%} + {% if existing_relation is none or full_refresh_mode or existing_relation.is_view %} - {% elif existing_relation.is_view %} - {#-- Can't overwrite a view with a table - we must drop --#} - {{ log("Dropping relation " ~ target_relation ~ " because it is a view and this model is a table.") }} - {{ adapter.drop_relation(existing_relation) }} + {% set tmp_vw_relation = target_relation.incorporate(path={"identifier": target_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_vw_relation) }} {%- call statement('main') -%} {{ get_create_table_as_sql(False, target_relation, sql)}} {%- endcall -%} - {% elif full_refresh_mode %} - {%- call statement('main') -%} - {{ get_create_table_as_sql(False, target_relation, sql)}} - {%- endcall -%} + -- Dropping temp view relation + {{ adapter.drop_relation(tmp_vw_relation) }} {% else %} + + {%- set temp_relation = make_temp_relation(target_relation)-%} + {% set tmp_tble_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_tble_vw_relation) }} + {%- call statement('create_tmp_relation') -%} {{ get_create_table_as_sql(True, temp_relation, sql)}} {%- endcall -%} + {{ adapter.drop_relation(tmp_tble_vw_relation) }} {% do adapter.expand_target_column_types( from_relation=temp_relation, to_relation=target_relation) %} @@ -59,9 +62,10 @@ {%- call statement('main') -%} {{ strategy_sql_macro_func(strategy_arg_dict) }} {%- endcall -%} + + {{ adapter.drop_relation(temp_relation) }} {% endif %} - {{ adapter.drop_relation(temp_relation) }} {{ run_hooks(post_hooks, inside_transaction=True) }} {% set target_relation = target_relation.incorporate(type='table') %} {% set should_revoke = should_revoke(existing_relation, full_refresh_mode) %} diff --git a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql index 8ed7ae2..cbef0c9 100644 --- a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql @@ -1,7 +1,6 @@ {% macro fabric__create_table_as(temporary, relation, sql) -%} - {% set tmp_vw_relation = relation.incorporate(path={"identifier": relation.identifier ~ '_vw'}, type='view')-%} - {% do adapter.drop_relation(tmp_vw_relation) %} + {% set tmp_vw_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} {{ get_create_view_as_sql(tmp_vw_relation, sql) }} {% set contract_config = config.get('contract') %} @@ -22,6 +21,6 @@ {%- else %} EXEC('CREATE TABLE {{relation}} AS SELECT * FROM {{tmp_vw_relation}};'); {% endif %} - + -- To clean up test_store tests {% do adapter.drop_relation(tmp_vw_relation) %} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/models/table/table.sql b/dbt/include/fabric/macros/materializations/models/table/table.sql index 3431761..4eb1f4a 100644 --- a/dbt/include/fabric/macros/materializations/models/table/table.sql +++ b/dbt/include/fabric/macros/materializations/models/table/table.sql @@ -20,7 +20,9 @@ -- Drop temp relation if it exists before materializing temp relation {{ adapter.drop_relation(temp_relation) }} - {% set tmp_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '_vw'}, type='view')-%} + {% set tmp_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + + {{ adapter.drop_relation(tmp_vw_relation) }} {{ run_hooks(pre_hooks, inside_transaction=False) }} -- `BEGIN` happens here: diff --git a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql index a37ae64..e1c8f83 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql @@ -191,9 +191,17 @@ {% macro build_snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} {% set temp_relation = make_temp_relation(target_relation) %} {% set select = snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} + + {% set tmp_tble_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_tble_vw_relation) }} + {% call statement('build_snapshot_staging_relation') %} {{ get_create_table_as_sql(True, temp_relation, select) }} {% endcall %} + -- Dropping temp view relation if it exists + {{ adapter.drop_relation(tmp_tble_vw_relation) }} + {% do return(temp_relation) %} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql b/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql index bf874bb..07c6789 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/snapshot.sql @@ -44,9 +44,9 @@ {% set tmp_relation_view = target_relation.incorporate(path={"identifier": target_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} -- Fabric & Synapse adapters use temp relation because of lack of CTE support for CTE in CTAS, Insert -- drop temp relation if exists - {% do adapter.drop_relation(tmp_relation_view) %} + {{ adapter.drop_relation(tmp_relation_view) }} {% set final_sql = get_create_table_as_sql(False, target_relation, build_sql) %} - {% do adapter.drop_relation(tmp_relation_view) %} + {{ adapter.drop_relation(tmp_relation_view) }} {% else %} @@ -88,7 +88,7 @@ {{ final_sql }} {% endcall %} - {% do adapter.drop_relation(temp_snapshot_relation) %} + {{ adapter.drop_relation(temp_snapshot_relation) }} {% set should_revoke = should_revoke(target_relation_exists, full_refresh_mode=False) %} {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} From 6f5acc5729a1800b7f205ebae82d6f51f6ba59bd Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 21 Jun 2024 01:35:34 -0700 Subject: [PATCH 04/10] Addressing #179, incremental models cannot full refresh --- .../macros/materializations/models/incremental/incremental.sql | 3 +++ .../macros/materializations/models/table/create_table_as.sql | 3 +-- .../fabric/macros/materializations/snapshots/helpers.sql | 2 ++ 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql b/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql index 4599ba2..bb7d4b3 100644 --- a/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql +++ b/dbt/include/fabric/macros/materializations/models/incremental/incremental.sql @@ -27,6 +27,8 @@ {% set tmp_vw_relation = target_relation.incorporate(path={"identifier": target_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} -- Dropping temp view relation if it exists {{ adapter.drop_relation(tmp_vw_relation) }} + -- Dropping target relation if exists + {{ adapter.drop_relation(target_relation) }} {%- call statement('main') -%} {{ get_create_table_as_sql(False, target_relation, sql)}} @@ -38,6 +40,7 @@ {% else %} {%- set temp_relation = make_temp_relation(target_relation)-%} + {{ adapter.drop_relation(temp_relation) }} {% set tmp_tble_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} -- Dropping temp view relation if it exists {{ adapter.drop_relation(tmp_tble_vw_relation) }} diff --git a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql index cbef0c9..139147f 100644 --- a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql @@ -1,6 +1,7 @@ {% macro fabric__create_table_as(temporary, relation, sql) -%} {% set tmp_vw_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} + {% do adapter.drop_relation(tmp_vw_relation) %} {{ get_create_view_as_sql(tmp_vw_relation, sql) }} {% set contract_config = config.get('contract') %} @@ -21,6 +22,4 @@ {%- else %} EXEC('CREATE TABLE {{relation}} AS SELECT * FROM {{tmp_vw_relation}};'); {% endif %} - -- To clean up test_store tests - {% do adapter.drop_relation(tmp_vw_relation) %} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql index e1c8f83..3d7583a 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql @@ -190,6 +190,8 @@ {% macro build_snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} {% set temp_relation = make_temp_relation(target_relation) %} + {{ adapter.drop_relation(temp_relation) }} + {% set select = snapshot_staging_table(strategy, temp_snapshot_relation, target_relation) %} {% set tmp_tble_vw_relation = temp_relation.incorporate(path={"identifier": temp_relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} From 8f0f0d55de2541d23c31a563de9270b93fd1244a Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sat, 22 Jun 2024 16:21:49 -0700 Subject: [PATCH 05/10] Adding Query label support to filter queries executd by dbt-fabric adapter --- .../fabric/macros/adapters/catalog.sql | 243 +++++++++--------- .../fabric/macros/adapters/columns.sql | 59 +++-- .../fabric/macros/adapters/metadata.sql | 15 +- .../fabric/macros/adapters/relation.sql | 1 + .../models/incremental/merge.sql | 11 +- .../models/table/create_table_as.sql | 7 +- .../models/view/create_view_as.sql | 1 - .../macros/materializations/seeds/helpers.sql | 1 + .../materializations/snapshots/helpers.sql | 4 +- .../snapshots/snapshot_merge.sql | 11 +- tests/functional/adapter/test_constraints.py | 10 +- 11 files changed, 194 insertions(+), 169 deletions(-) diff --git a/dbt/include/fabric/macros/adapters/catalog.sql b/dbt/include/fabric/macros/adapters/catalog.sql index cd02a29..6bd1398 100644 --- a/dbt/include/fabric/macros/adapters/catalog.sql +++ b/dbt/include/fabric/macros/adapters/catalog.sql @@ -1,131 +1,132 @@ {% macro fabric__get_catalog(information_schemas, schemas) -%} + {% set query_label = apply_label() %} + {%- call statement('catalog', fetch_result=True) -%} - {%- call statement('catalog', fetch_result=True) -%} - - with - principals as ( - select - name as principal_name, - principal_id as principal_id - from - sys.database_principals {{ information_schema_hints() }} - ), - - schemas as ( - select - name as schema_name, - schema_id as schema_id, - principal_id as principal_id - from - sys.schemas {{ information_schema_hints() }} - ), - - tables as ( - select - object_id, - name as table_name, - schema_id as schema_id, - principal_id as principal_id, - 'BASE TABLE' as table_type - from - sys.tables {{ information_schema_hints() }} - ), - - tables_with_metadata as ( - select - object_id, - table_name, - schema_name, - coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id, - table_type - from - tables - join schemas on tables.schema_id = schemas.schema_id - ), - - views as ( - select - object_id, - name as table_name, - schema_id as schema_id, - principal_id as principal_id, - 'VIEW' as table_type - from - sys.views {{ information_schema_hints() }} - ), - - views_with_metadata as ( - select - object_id, - table_name, - schema_name, - coalesce(views.principal_id, schemas.principal_id) as owner_principal_id, - table_type - from - views - join schemas on views.schema_id = schemas.schema_id - ), - - tables_and_views as ( - select - object_id, - table_name, - schema_name, - principal_name, - table_type - from - tables_with_metadata - join principals on tables_with_metadata.owner_principal_id = principals.principal_id - union all - select - object_id, - table_name, - schema_name, - principal_name, - table_type - from - views_with_metadata - join principals on views_with_metadata.owner_principal_id = principals.principal_id - ), - - cols as ( + with + principals as ( + select + name as principal_name, + principal_id as principal_id + from + sys.database_principals {{ information_schema_hints() }} + ), + + schemas as ( + select + name as schema_name, + schema_id as schema_id, + principal_id as principal_id + from + sys.schemas {{ information_schema_hints() }} + ), + + tables as ( + select + object_id, + name as table_name, + schema_id as schema_id, + principal_id as principal_id, + 'BASE TABLE' as table_type + from + sys.tables {{ information_schema_hints() }} + ), + + tables_with_metadata as ( + select + object_id, + table_name, + schema_name, + coalesce(tables.principal_id, schemas.principal_id) as owner_principal_id, + table_type + from + tables + join schemas on tables.schema_id = schemas.schema_id + ), + + views as ( + select + object_id, + name as table_name, + schema_id as schema_id, + principal_id as principal_id, + 'VIEW' as table_type + from + sys.views {{ information_schema_hints() }} + ), + + views_with_metadata as ( + select + object_id, + table_name, + schema_name, + coalesce(views.principal_id, schemas.principal_id) as owner_principal_id, + table_type + from + views + join schemas on views.schema_id = schemas.schema_id + ), + + tables_and_views as ( + select + object_id, + table_name, + schema_name, + principal_name, + table_type + from + tables_with_metadata + join principals on tables_with_metadata.owner_principal_id = principals.principal_id + union all + select + object_id, + table_name, + schema_name, + principal_name, + table_type + from + views_with_metadata + join principals on views_with_metadata.owner_principal_id = principals.principal_id + ), + + cols as ( + + select + c.object_id, + c.name as column_name, + c.column_id as column_index, + t.name as column_type + from sys.columns as c {{ information_schema_hints() }} + left join sys.types as t on c.system_type_id = t.system_type_id {{ information_schema_hints() }} + ) select - c.object_id, - c.name as column_name, - c.column_id as column_index, - t.name as column_type - from sys.columns as c {{ information_schema_hints() }} - left join sys.types as t on c.system_type_id = t.system_type_id {{ information_schema_hints() }} - ) - - select - DB_NAME() as table_database, - tv.schema_name as table_schema, - tv.table_name, - tv.table_type, - null as table_comment, - tv.principal_name as table_owner, - cols.column_name, - cols.column_index, - cols.column_type, - null as column_comment - from tables_and_views tv - join cols on tv.object_id = cols.object_id - where ({%- for schema in schemas -%} - upper(tv.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} - {%- endfor -%}) - - order by column_index - - {%- endcall -%} - - {{ return(load_result('catalog').table) }} + DB_NAME() as table_database, + tv.schema_name as table_schema, + tv.table_name, + tv.table_type, + null as table_comment, + tv.principal_name as table_owner, + cols.column_name, + cols.column_index, + cols.column_type, + null as column_comment + from tables_and_views tv + join cols on tv.object_id = cols.object_id + where ({%- for schema in schemas -%} + upper(tv.schema_name) = upper('{{ schema }}'){%- if not loop.last %} or {% endif -%} + {%- endfor -%}) + + order by column_index + {{ query_label }} + + {%- endcall -%} + + {{ return(load_result('catalog').table) }} {%- endmacro %} {% macro fabric__get_catalog_relations(information_schema, relations) -%} - + {% set query_label = apply_label() %} {%- call statement('catalog', fetch_result=True) -%} with @@ -260,7 +261,7 @@ ) order by column_index - + {{ query_label }} {%- endcall -%} {{ return(load_result('catalog').table) }} diff --git a/dbt/include/fabric/macros/adapters/columns.sql b/dbt/include/fabric/macros/adapters/columns.sql index 69f2ce8..26c8fc2 100644 --- a/dbt/include/fabric/macros/adapters/columns.sql +++ b/dbt/include/fabric/macros/adapters/columns.sql @@ -11,42 +11,46 @@ {% endmacro %} {% macro fabric__get_columns_in_relation(relation) -%} - {% call statement('get_columns_in_relation', fetch_result=True) %} + {% set query_label = apply_label() %} + {% call statement('get_columns_in_relation', fetch_result=True) %} + + with mapping as ( + select + row_number() over (partition by object_name(c.object_id) order by c.column_id) as ordinal_position, + c.name collate database_default as column_name, + t.name as data_type, + c.max_length as character_maximum_length, + c.precision as numeric_precision, + c.scale as numeric_scale + from [{{ 'tempdb' if '#' in relation.identifier else relation.database }}].sys.columns c {{ information_schema_hints() }} + inner join sys.types t {{ information_schema_hints() }} + on c.user_type_id = t.user_type_id + where c.object_id = object_id('{{ 'tempdb..' ~ relation.include(database=false, schema=false) if '#' in relation.identifier else relation }}') + ) - with mapping as ( select - row_number() over (partition by object_name(c.object_id) order by c.column_id) as ordinal_position, - c.name collate database_default as column_name, - t.name as data_type, - c.max_length as character_maximum_length, - c.precision as numeric_precision, - c.scale as numeric_scale - from [{{ 'tempdb' if '#' in relation.identifier else relation.database }}].sys.columns c {{ information_schema_hints() }} - inner join sys.types t {{ information_schema_hints() }} - on c.user_type_id = t.user_type_id - where c.object_id = object_id('{{ 'tempdb..' ~ relation.include(database=false, schema=false) if '#' in relation.identifier else relation }}') - ) - - select - column_name, - data_type, - character_maximum_length, - numeric_precision, - numeric_scale - from mapping - order by ordinal_position - - {% endcall %} - {% set table = load_result('get_columns_in_relation').table %} - {{ return(sql_convert_columns_in_relation(table)) }} + column_name, + data_type, + character_maximum_length, + numeric_precision, + numeric_scale + from mapping + order by ordinal_position + {{ query_label }} + + {% endcall %} + {% set table = load_result('get_columns_in_relation').table %} + {{ return(sql_convert_columns_in_relation(table)) }} {% endmacro %} {% macro fabric__get_columns_in_query(select_sql) %} + {% set query_label = apply_label() %} {% call statement('get_columns_in_query', fetch_result=True, auto_begin=False) -%} select TOP 0 * from ( {{ select_sql }} ) as __dbt_sbq where 0 = 1 + {{ query_label }} {% endcall %} {{ return(load_result('get_columns_in_query').table.columns | map(attribute='name') | list) }} @@ -84,6 +88,7 @@ {% set tempTable %} CREATE TABLE {{tempTableName}} AS SELECT {{query_result_text}}, CAST({{ column_name }} AS {{new_column_type}}) AS {{column_name}} FROM {{ relation.schema }}.{{ relation.identifier }} + {{ apply_label() }} {% endset %} {% call statement('create_temp_table') -%} @@ -100,7 +105,7 @@ {% set createTable %} CREATE TABLE {{ relation.schema }}.{{ relation.identifier }} - AS SELECT * FROM {{tempTableName}} + AS SELECT * FROM {{tempTableName}} {{ apply_label() }} {% endset %} {% call statement('create_Table') -%} diff --git a/dbt/include/fabric/macros/adapters/metadata.sql b/dbt/include/fabric/macros/adapters/metadata.sql index 2fe6a58..4d4f9fa 100644 --- a/dbt/include/fabric/macros/adapters/metadata.sql +++ b/dbt/include/fabric/macros/adapters/metadata.sql @@ -1,3 +1,10 @@ + +{% macro apply_label() %} + {{ log (config.get('query_tag','dbt-fabric'))}} + {%- set query_label = config.get('query_tag','dbt-fabric-dw') -%} + OPTION (LABEL = '{{query_label}}'); +{% endmacro %} + {% macro information_schema_hints() %} {{ return(adapter.dispatch('information_schema_hints')()) }} {% endmacro %} @@ -20,15 +27,14 @@ {% macro fabric__list_schemas(database) %} {% call statement('list_schemas', fetch_result=True, auto_begin=False) -%} select name as [schema] - from sys.schemas {{ information_schema_hints() }} + from sys.schemas {{ information_schema_hints() }} {{ apply_label() }} {% endcall %} {{ return(load_result('list_schemas').table) }} {% endmacro %} {% macro fabric__check_schema_exists(information_schema, schema) -%} {% call statement('check_schema_exists', fetch_result=True, auto_begin=False) -%} - - SELECT count(*) as schema_exist FROM sys.schemas WHERE name = '{{ schema }}' + SELECT count(*) as schema_exist FROM sys.schemas WHERE name = '{{ schema }}' {{ apply_label() }} {%- endcall %} {{ return(load_result('check_schema_exists').table) }} {% endmacro %} @@ -53,6 +59,7 @@ ) select * from base where [schema] like '{{ schema_relation.schema }}' + {{ apply_label() }} {% endcall %} {{ return(load_result('list_relations_without_caching').table) }} {% endmacro %} @@ -78,6 +85,7 @@ select * from base where [schema] like '{{ schema_relation.schema }}' and [name] like '{{ schema_relation.identifier }}' + {{ apply_label() }} {% endcall %} {{ return(load_result('get_relation_without_caching').table) }} {% endmacro %} @@ -97,6 +105,7 @@ upper(o.name) = upper('{{ relation.identifier }}')){%- if not loop.last %} or {% endif -%} {%- endfor -%} ) + {{ apply_label() }} {%- endcall -%} {{ return(load_result('last_modified')) }} diff --git a/dbt/include/fabric/macros/adapters/relation.sql b/dbt/include/fabric/macros/adapters/relation.sql index 85aa0e2..6eac9bb 100644 --- a/dbt/include/fabric/macros/adapters/relation.sql +++ b/dbt/include/fabric/macros/adapters/relation.sql @@ -23,6 +23,7 @@ and refs.referenced_entity_name = '{{ relation.identifier }}' and refs.referencing_class = 1 and obj.type = 'V' + {{ apply_label() }} {% endcall %} {% set references = load_result('find_references')['data'] %} {% for reference in references -%} diff --git a/dbt/include/fabric/macros/materializations/models/incremental/merge.sql b/dbt/include/fabric/macros/materializations/models/incremental/merge.sql index 6f4315a..b5a0903 100644 --- a/dbt/include/fabric/macros/materializations/models/incremental/merge.sql +++ b/dbt/include/fabric/macros/materializations/models/incremental/merge.sql @@ -14,6 +14,8 @@ {% endmacro %} {% macro fabric__get_delete_insert_merge_sql(target, source, unique_key, dest_columns, incremental_predicates=none) %} + + {% set query_label = apply_label() %} {%- set dest_cols_csv = get_quoted_csv(dest_columns | map(attribute="name")) -%} {% if unique_key %} @@ -27,13 +29,13 @@ {{ source }}.{{ key }} = {{ target }}.{{ key }} {{ "and " if not loop.last }} {% endfor %} - ) {% if incremental_predicates %} {% for predicate in incremental_predicates %} and {{ predicate }} {% endfor %} - {% endif %}; + {% endif %} + {{ query_label }} {% else %} delete from {{ target }} where ( @@ -45,7 +47,8 @@ {% for predicate in incremental_predicates %} and {{ predicate }} {% endfor %} - {%- endif -%}; + {%- endif -%} + {{ query_label }} {% endif %} {% endif %} @@ -53,5 +56,5 @@ ( select {{ dest_cols_csv }} from {{ source }} - ) + ){{ query_label }} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql index 139147f..e2a946a 100644 --- a/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql +++ b/dbt/include/fabric/macros/materializations/models/table/create_table_as.sql @@ -1,5 +1,5 @@ {% macro fabric__create_table_as(temporary, relation, sql) -%} - + {% set query_label = apply_label() %} {% set tmp_vw_relation = relation.incorporate(path={"identifier": relation.identifier ~ '__dbt_tmp_vw'}, type='view')-%} {% do adapter.drop_relation(tmp_vw_relation) %} {{ get_create_view_as_sql(tmp_vw_relation, sql) }} @@ -17,9 +17,10 @@ {%endset%} INSERT INTO {{relation}} ({{listColumns}}) - SELECT {{listColumns}} FROM {{tmp_vw_relation}}; + SELECT {{listColumns}} FROM {{tmp_vw_relation}} {{ query_label }} {%- else %} - EXEC('CREATE TABLE {{relation}} AS SELECT * FROM {{tmp_vw_relation}};'); + {%- set query_label_option = query_label.replace("'", "''") -%} + EXEC('CREATE TABLE {{relation}} AS SELECT * FROM {{tmp_vw_relation}} {{ query_label_option }}'); {% endif %} {% endmacro %} diff --git a/dbt/include/fabric/macros/materializations/models/view/create_view_as.sql b/dbt/include/fabric/macros/materializations/models/view/create_view_as.sql index f0ec12d..cf63055 100644 --- a/dbt/include/fabric/macros/materializations/models/view/create_view_as.sql +++ b/dbt/include/fabric/macros/materializations/models/view/create_view_as.sql @@ -5,7 +5,6 @@ {% macro fabric__create_view_exec(relation, sql) -%} {%- set temp_view_sql = sql.replace("'", "''") -%} - {{ get_use_database_sql(relation.database) }} {% set contract_config = config.get('contract') %} {% if contract_config.enforced %} diff --git a/dbt/include/fabric/macros/materializations/seeds/helpers.sql b/dbt/include/fabric/macros/materializations/seeds/helpers.sql index 68d5f80..4526ccd 100644 --- a/dbt/include/fabric/macros/materializations/seeds/helpers.sql +++ b/dbt/include/fabric/macros/materializations/seeds/helpers.sql @@ -43,6 +43,7 @@ {%- endfor -%}) {%- if not loop.last%},{%- endif %} {%- endfor %} + {{ apply_label()}} {% endset %} {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %} diff --git a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql index 3d7583a..bf7c77a 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/helpers.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/helpers.sql @@ -22,7 +22,7 @@ {% set tempTable %} CREATE TABLE {{tempTableName}} - AS SELECT * {{columns}} FROM [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] {{ information_schema_hints() }} + AS SELECT * {{columns}} FROM [{{relation.database}}].[{{ relation.schema }}].[{{ relation.identifier }}] {{ information_schema_hints() }} {{ apply_label() }} {% endset %} {% call statement('create_temp_table') -%} @@ -39,7 +39,7 @@ {% set createTable %} CREATE TABLE {{ relation }} - AS SELECT * FROM {{tempTableName}} {{ information_schema_hints() }} + AS SELECT * FROM {{tempTableName}} {{ information_schema_hints() }} {{ apply_label() }} {% endset %} {% call statement('create_Table') -%} diff --git a/dbt/include/fabric/macros/materializations/snapshots/snapshot_merge.sql b/dbt/include/fabric/macros/materializations/snapshots/snapshot_merge.sql index 838da43..e2d3e99 100644 --- a/dbt/include/fabric/macros/materializations/snapshots/snapshot_merge.sql +++ b/dbt/include/fabric/macros/materializations/snapshots/snapshot_merge.sql @@ -1,15 +1,12 @@ {% macro fabric__snapshot_merge_sql(target, source, insert_cols) %} - {%- set insert_cols_csv = insert_cols | join(', ') -%} + {%- set insert_cols_csv = insert_cols | join(', ') -%} {%- set target_table = target.include(database=False) -%} {%- set source_table = source.include(database=False) -%} - {% set target_columns_list = [] %} - {% for column in insert_cols %} {% set target_columns_list = target_columns_list.append("DBT_INTERNAL_SOURCE."+column) %} {% endfor %} - {%- set target_columns = target_columns_list | join(', ') -%} UPDATE DBT_INTERNAL_DEST @@ -18,9 +15,11 @@ INNER JOIN {{ source_table }} as DBT_INTERNAL_SOURCE on DBT_INTERNAL_SOURCE.dbt_scd_id = DBT_INTERNAL_DEST.dbt_scd_id WHERE DBT_INTERNAL_DEST.dbt_valid_to is null - AND DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete'); + AND DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') + {{ apply_label() }} INSERT INTO {{ target_table }} ({{ insert_cols_csv }}) SELECT {{target_columns}} FROM {{ source_table }} as DBT_INTERNAL_SOURCE - WHERE DBT_INTERNAL_SOURCE.dbt_change_type = 'insert'; + WHERE DBT_INTERNAL_SOURCE.dbt_change_type = 'insert' + {{ apply_label() }} {% endmacro %} diff --git a/tests/functional/adapter/test_constraints.py b/tests/functional/adapter/test_constraints.py index a72a358..9168c0c 100644 --- a/tests/functional/adapter/test_constraints.py +++ b/tests/functional/adapter/test_constraints.py @@ -503,8 +503,11 @@ def test__constraints_ddl(self, project, expected_sql): generated_sql_generic, "foreign_key_model", "" ) generated_sql_wodb = generated_sql_generic.replace("USE [" + project.database + "];", "") + generated_sql_option_cluase = generated_sql_wodb.replace( + "OPTION (LABEL = 'dbt-fabric-dw');", "" + ) assert _normalize_whitespace(expected_sql.strip()) == _normalize_whitespace( - generated_sql_wodb.strip() + generated_sql_option_cluase.strip() ) @@ -568,8 +571,11 @@ def test__model_constraints_ddl(self, project, expected_sql): generated_sql_generic, "foreign_key_model", "" ) generated_sql_wodb = generated_sql_generic.replace("USE [" + project.database + "];", "") + generated_sql_option_cluase = generated_sql_wodb.replace( + "OPTION (LABEL = 'dbt-fabric-dw');", "" + ) assert _normalize_whitespace(expected_sql.strip()) == _normalize_whitespace( - generated_sql_wodb.strip() + generated_sql_option_cluase.strip() ) From 69e6df2a8efe937777a2ade1b014c49eef22c238 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sat, 22 Jun 2024 20:48:04 -0700 Subject: [PATCH 06/10] Updating dependencies --- .github/workflows/publish-docker.yml | 4 ++-- dev_requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 18fd7fa..763baba 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -24,14 +24,14 @@ jobs: uses: actions/checkout@v4 - name: Log in to the Container registry - uses: docker/login-action@v3.0.0 + uses: docker/login-action@v3.2.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Docker image - uses: docker/build-push-action@v5.1.0 + uses: docker/build-push-action@v6.1.0 with: context: devops build-args: PYTHON_VERSION=${{ matrix.python_version }} diff --git a/dev_requirements.txt b/dev_requirements.txt index a42c6b9..cb00c65 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -5,7 +5,7 @@ git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git -pytest==8.0.1 +pytest==8.2.2 twine==5.0.0 wheel==0.42 pre-commit==3.5.0;python_version<"3.9" From 3ecedc21f19c3c5845f6bd3c0c124a897c6f7851 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sat, 22 Jun 2024 20:53:33 -0700 Subject: [PATCH 07/10] Updating dependencies --- .github/workflows/publish-docker.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 763baba..48c4b9f 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -24,14 +24,14 @@ jobs: uses: actions/checkout@v4 - name: Log in to the Container registry - uses: docker/login-action@v3.2.0 + uses: docker/login-action@v3.3.0 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push Docker image - uses: docker/build-push-action@v6.1.0 + uses: docker/build-push-action@v5.1.0 with: context: devops build-args: PYTHON_VERSION=${{ matrix.python_version }} From 64b90994b0742b7f5779485bf14483e3d7cb89f1 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sat, 22 Jun 2024 20:55:25 -0700 Subject: [PATCH 08/10] Updating dependencies --- dev_requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index cb00c65..a42c6b9 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -5,7 +5,7 @@ git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git -pytest==8.2.2 +pytest==8.0.1 twine==5.0.0 wheel==0.42 pre-commit==3.5.0;python_version<"3.9" From 7b67d8cf41a42bab3407e92b4b80e5c9ae36bc8c Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sat, 22 Jun 2024 21:09:11 -0700 Subject: [PATCH 09/10] Updating change log --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d641e2c..778c9e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +### V1.8.7 +* Improving table materialization to minimize downtime #189 +* Handling temp tables in incremental models #188 +* Add label support to filter queries #181 +* Addressed bug - incremental models cannot full refresh #179 + ### v1.8.0rc2 ## Bug Fixes From faa2cb0ddbc1476735cbb86f582d01bbf17f680f Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Wed, 26 Jun 2024 21:49:31 -0700 Subject: [PATCH 10/10] Addressed #197 --- CHANGELOG.md | 1 + dbt/include/fabric/macros/materializations/tests/helpers.sql | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 778c9e6..f300748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * Handling temp tables in incremental models #188 * Add label support to filter queries #181 * Addressed bug - incremental models cannot full refresh #179 +* Addressed bug - #197, dbt test incorrect syntax with macro helpers.sql ### v1.8.0rc2 diff --git a/dbt/include/fabric/macros/materializations/tests/helpers.sql b/dbt/include/fabric/macros/materializations/tests/helpers.sql index 9e4b057..4f04547 100644 --- a/dbt/include/fabric/macros/materializations/tests/helpers.sql +++ b/dbt/include/fabric/macros/materializations/tests/helpers.sql @@ -9,7 +9,7 @@ {% if main_sql.strip().lower().startswith('with') %} {% set testview %} - {{ target.schema }}.testview_{{ range(1300, 19000) | random }} + [{{ target.schema }}.testview_{{ range(1300, 19000) | random }}] {% endset %} {% set sql = main_sql.replace("'", "''")%}