From befa7b52a03bdd47329bbef13efb03b3646af91a Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Thu, 21 Nov 2024 17:06:48 -0500 Subject: [PATCH 01/13] add callout for net new --- website/docs/reference/snapshot-configs.md | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 7b3c0f8e5b1..22c5ebf86cc 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -13,11 +13,6 @@ import ConfigGeneral from '/snippets/_config-description-general.md'; * [Snapshots](/docs/build/snapshots) * The `dbt snapshot` [command](/reference/commands/snapshot) - ## Available configurations ### Snapshot-specific configurations @@ -125,7 +120,7 @@ snapshots: -Configurations can be applied to snapshots using the [YAML syntax](/docs/build/snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. +Configurations can be applied to snapshots using the [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. @@ -155,6 +150,17 @@ Configurations can be applied to snapshots using the [YAML syntax](/docs/build/s +:::tip + +Configuring snapshots in YAML is recommended for new snapshot configurations. If you have existing snapshots using the `.sql` config, they'll continue to work as expected. When you're ready to migrate to YAML, we recommend: +1. Creating a backup copy of your snapshots (or using version control) +2. Converting the configurations one at a time, testing as you go +3. Using `alter` statements as needed to ensure table consistency or using a script to apply the `alter` statements + +The YAML configuration offers improved maintainability and consistency, but there's no pressure to migrate existing snapshots immediately — feel free to do so at your own pace while ensuring data quality is maintained. +::: + + + ```yaml snapshots: [](/reference/resource-configs/resource-path): @@ -256,7 +263,7 @@ snapshots: -Configurations can be applied to snapshots using [YAML syntax](/docs/build/snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. +Configurations can be applied to snapshots using [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. From 0347fd3d174d81b5b738976b82a97e26e722bf66 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Mon, 25 Nov 2024 16:20:27 +0000 Subject: [PATCH 02/13] turn to snippet --- website/docs/reference/snapshot-configs.md | 6 ++++-- website/snippets/_snapshots-yaml-config.md | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) create mode 100644 website/snippets/_snapshots-yaml-config.md diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 22c5ebf86cc..4a3541c2d51 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -7,6 +7,7 @@ meta: import ConfigResource from '/snippets/_config-description-resource.md'; import ConfigGeneral from '/snippets/_config-description-general.md'; +import SnapshotsYamlConfig from '/snippets/_snapshots-yaml-config.md'; ## Related documentation @@ -120,7 +121,8 @@ snapshots: -Configurations can be applied to snapshots using the [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. + + @@ -263,7 +265,7 @@ snapshots: -Configurations can be applied to snapshots using [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. + diff --git a/website/snippets/_snapshots-yaml-config.md b/website/snippets/_snapshots-yaml-config.md new file mode 100644 index 00000000000..68ccd7c0bc2 --- /dev/null +++ b/website/snippets/_snapshots-yaml-config.md @@ -0,0 +1,2 @@ +Configurations can be applied to snapshots using the [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. + From 576af99f2d7654c733a884e415a3629e628ff4ae Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Mon, 25 Nov 2024 16:22:10 +0000 Subject: [PATCH 03/13] Update website/docs/reference/snapshot-configs.md Co-authored-by: Grace Goheen <53586774+graciegoheen@users.noreply.github.com> --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 4a3541c2d51..5f5e88c9776 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -154,7 +154,7 @@ snapshots: :::tip -Configuring snapshots in YAML is recommended for new snapshot configurations. If you have existing snapshots using the `.sql` config, they'll continue to work as expected. When you're ready to migrate to YAML, we recommend: +Configuring snapshots in YAML is recommended for new snapshots. If you have existing snapshots using the `.sql` config, they'll continue to work as expected. When you're ready to migrate to YAML, we recommend: 1. Creating a backup copy of your snapshots (or using version control) 2. Converting the configurations one at a time, testing as you go 3. Using `alter` statements as needed to ensure table consistency or using a script to apply the `alter` statements From 2532cad5b35d9c6ebd0d3e8e964eb1d00ebdd34d Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Mon, 25 Nov 2024 16:24:41 +0000 Subject: [PATCH 04/13] update grace feedback --- website/docs/reference/snapshot-configs.md | 2 +- website/snippets/_snapshots-yaml-config.md | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 4a3541c2d51..e9ba021403d 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -155,7 +155,7 @@ snapshots: :::tip Configuring snapshots in YAML is recommended for new snapshot configurations. If you have existing snapshots using the `.sql` config, they'll continue to work as expected. When you're ready to migrate to YAML, we recommend: -1. Creating a backup copy of your snapshots (or using version control) +1. Creating a backup copy of your snapshots. 2. Converting the configurations one at a time, testing as you go 3. Using `alter` statements as needed to ensure table consistency or using a script to apply the `alter` statements diff --git a/website/snippets/_snapshots-yaml-config.md b/website/snippets/_snapshots-yaml-config.md index 68ccd7c0bc2..10eb8fed17c 100644 --- a/website/snippets/_snapshots-yaml-config.md +++ b/website/snippets/_snapshots-yaml-config.md @@ -1,2 +1 @@ -Configurations can be applied to snapshots using the [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. - +Define snapshots using the latest [YAML syntax](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. From 7e68e914ffcd179b0b04ff28250fec5d42c61aaa Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Wed, 27 Nov 2024 11:20:11 +0000 Subject: [PATCH 05/13] update to snapshot configs --- website/docs/reference/snapshot-configs.md | 30 ++++++++++++------- website/snippets/_snapshots-yaml-config.md | 2 +- .../components/expandable/styles.module.css | 3 +- 3 files changed, 22 insertions(+), 13 deletions(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index a2e8365ae93..fbcdad93d56 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -18,6 +18,7 @@ import SnapshotsYamlConfig from '/snippets/_snapshots-yaml-config.md'; ## Available configurations ### Snapshot-specific configurations + @@ -74,8 +75,9 @@ snapshots: [+](/reference/resource-configs/plus-prefix)[strategy](/reference/resource-configs/strategy): timestamp | check [+](/reference/resource-configs/plus-prefix)[updated_at](/reference/resource-configs/updated_at): [+](/reference/resource-configs/plus-prefix)[check_cols](/reference/resource-configs/check_cols): [] | all + [+](/reference/resource-configs/plus-prefix)[invalidate_hard_deletes](/reference/resource-configs/invalidate_hard_deletes) : true | false [+](/reference/resource-configs/plus-prefix)[snapshot_meta_column_names](/reference/resource-configs/snapshot_meta_column_names): {} - [+](/reference/resource-configs/plus-prefix)[invalidate_hard_deletes](/reference/resource-configs/invalidate_hard_deletes) : true | false + [+](/reference/resource-configs/plus-prefix)[dbt_valid_to_current](/reference/resource-configs/dbt_valid_to_current): ``` @@ -108,8 +110,9 @@ snapshots: [strategy](/reference/resource-configs/strategy): timestamp | check [updated_at](/reference/resource-configs/updated_at): [check_cols](/reference/resource-configs/check_cols): [] | all - [snapshot_meta_column_names](/reference/resource-configs/snapshot_meta_column_names): {} [invalidate_hard_deletes](/reference/resource-configs/invalidate_hard_deletes) : true | false + [snapshot_meta_column_names](/reference/resource-configs/snapshot_meta_column_names): {} + [dbt_valid_to_current](/reference/resource-configs/dbt_valid_to_current): ``` @@ -147,20 +150,25 @@ snapshots: +### Snapshot configuration migration -### General configurations +The latest snapshot YAML configurations introduced in dbt v1.9 and higher (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names) and [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current)) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies to your snapshots: - +#### For new snapshots +- Use the latest snapshot YAML configurations when creating new snapshots that didn't exist previously. + +#### For existing snapshots +- Migrate tables — Migrate the previous snapshot to the new table schema and values: + - Create a backup copy of your snapshots. + - Use `alter` statements as needed to ensure table consistency or use a script to apply the `alter` statements. +- New YAML config — Convert the YAML configurations one at a time, testing as you go. -:::tip +If you use one of the latest configs, such as `dbt_valid_to_current`, without migrating your data may result in mixed old and new data, leading to incorrect downstream result. -Configuring snapshots in YAML is recommended for new snapshots. If you have existing snapshots using the `.sql` config, they'll continue to work as expected. When you're ready to migrate to YAML, we recommend: -1. Creating a backup copy of your snapshots. -2. Converting the configurations one at a time, testing as you go -3. Using `alter` statements as needed to ensure table consistency or using a script to apply the `alter` statements -The YAML configuration offers improved maintainability and consistency, but there's no pressure to migrate existing snapshots immediately — feel free to do so at your own pace while ensuring data quality is maintained. -::: +### General configurations + + Date: Wed, 27 Nov 2024 15:52:47 +0000 Subject: [PATCH 06/13] Update website/docs/reference/snapshot-configs.md Co-authored-by: Matt Shaver <60105315+matthewshaver@users.noreply.github.com> --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index fbcdad93d56..ad74ae9d423 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -163,7 +163,7 @@ The latest snapshot YAML configurations introduced in dbt v1.9 and higher (such - Use `alter` statements as needed to ensure table consistency or use a script to apply the `alter` statements. - New YAML config — Convert the YAML configurations one at a time, testing as you go. -If you use one of the latest configs, such as `dbt_valid_to_current`, without migrating your data may result in mixed old and new data, leading to incorrect downstream result. +If you use one of the latest configs, such as `dbt_valid_to_current`, without migrating your data, you may have mixed old and new data, leading to an incorrect downstream result. ### General configurations From bdf2850e5d4aae38d6d5c04a790952103803d6ce Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:53:03 +0000 Subject: [PATCH 07/13] Update website/docs/reference/snapshot-configs.md Co-authored-by: Matt Shaver <60105315+matthewshaver@users.noreply.github.com> --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index ad74ae9d423..a0edf569bf3 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -160,7 +160,7 @@ The latest snapshot YAML configurations introduced in dbt v1.9 and higher (such #### For existing snapshots - Migrate tables — Migrate the previous snapshot to the new table schema and values: - Create a backup copy of your snapshots. - - Use `alter` statements as needed to ensure table consistency or use a script to apply the `alter` statements. + - Use `alter` statements as needed (or a script to apply `alter` statements) to ensure table consistency. - New YAML config — Convert the YAML configurations one at a time, testing as you go. If you use one of the latest configs, such as `dbt_valid_to_current`, without migrating your data, you may have mixed old and new data, leading to an incorrect downstream result. From 6a84a7f955744ae9b52169acf2b9ec6a2dce532a Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:53:11 +0000 Subject: [PATCH 08/13] Update website/docs/reference/snapshot-configs.md Co-authored-by: Matt Shaver <60105315+matthewshaver@users.noreply.github.com> --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index a0edf569bf3..598a8a35aa4 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -152,7 +152,7 @@ snapshots: ### Snapshot configuration migration -The latest snapshot YAML configurations introduced in dbt v1.9 and higher (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names) and [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current)) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies to your snapshots: +The latest snapshot YAML configurations introduced in dbt v1.9 (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names) and [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current)) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies in your snapshots: #### For new snapshots - Use the latest snapshot YAML configurations when creating new snapshots that didn't exist previously. From 37efd7a83bf6ca39130ef66883495c5279dd2664 Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 27 Nov 2024 15:53:27 +0000 Subject: [PATCH 09/13] Update website/snippets/_snapshots-yaml-config.md Co-authored-by: Matt Shaver <60105315+matthewshaver@users.noreply.github.com> --- website/snippets/_snapshots-yaml-config.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/snippets/_snapshots-yaml-config.md b/website/snippets/_snapshots-yaml-config.md index cdfb7ceb07e..648b65752c0 100644 --- a/website/snippets/_snapshots-yaml-config.md +++ b/website/snippets/_snapshots-yaml-config.md @@ -1 +1 @@ -Define snapshots using the latest [YAML configurations](/docs/build/snapshots#configuring-snapshots), available in Versionless and dbt v1.9 and higher, in the `snapshot` directory file. +Define snapshots using the latest [YAML configurations](/docs/build/snapshots#configuring-snapshots), available in Versionless dbt Cloud and Core v1.9 and higher, in the `snapshot` directory file. From 5d3b48f6b12e90306403ca94d95fb9d0a991f318 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Thu, 28 Nov 2024 11:01:40 +0000 Subject: [PATCH 10/13] update --- website/docs/reference/snapshot-configs.md | 17 ++++++----------- website/snippets/_snapshots-yaml-config.md | 1 - 2 files changed, 6 insertions(+), 12 deletions(-) delete mode 100644 website/snippets/_snapshots-yaml-config.md diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 598a8a35aa4..4d8d8e42d26 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -18,14 +18,11 @@ import SnapshotsYamlConfig from '/snippets/_snapshots-yaml-config.md'; ## Available configurations ### Snapshot-specific configurations - -import SnapshotYaml from '/snippets/_snapshot-yaml-spec.md'; - - +Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshots#snapshot-meta-fields). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to migrate over. @@ -152,19 +149,17 @@ snapshots: ### Snapshot configuration migration -The latest snapshot YAML configurations introduced in dbt v1.9 (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names) and [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current)) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies in your snapshots: - -#### For new snapshots -- Use the latest snapshot YAML configurations when creating new snapshots that didn't exist previously. +The latest snapshot configurations introduced in dbt v1.9 (such as [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names), [`dbt_valid_to_current`](/reference/resource-configs/dbt_valid_to_current), and `hard_deletes`) are best suited for new snapshots. For existing snapshots, we recommend the following to avoid any inconsistencies in your snapshots: #### For existing snapshots - Migrate tables — Migrate the previous snapshot to the new table schema and values: - Create a backup copy of your snapshots. - Use `alter` statements as needed (or a script to apply `alter` statements) to ensure table consistency. -- New YAML config — Convert the YAML configurations one at a time, testing as you go. +- New configurations — Convert the configs one at a time, testing as you go. +:::warning If you use one of the latest configs, such as `dbt_valid_to_current`, without migrating your data, you may have mixed old and new data, leading to an incorrect downstream result. - +::: ### General configurations @@ -273,7 +268,7 @@ snapshots: - +Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [configurations](/docs/build/snapshots#configuring-snapshots). You can also continue using the legacy method for existing snapshots, but we recommend migrating over to the latest YAML format for better readability and maintainability. diff --git a/website/snippets/_snapshots-yaml-config.md b/website/snippets/_snapshots-yaml-config.md deleted file mode 100644 index 648b65752c0..00000000000 --- a/website/snippets/_snapshots-yaml-config.md +++ /dev/null @@ -1 +0,0 @@ -Define snapshots using the latest [YAML configurations](/docs/build/snapshots#configuring-snapshots), available in Versionless dbt Cloud and Core v1.9 and higher, in the `snapshot` directory file. From b84f4a5d97a09a428ce58bb94a5446d0fd6cbb28 Mon Sep 17 00:00:00 2001 From: mirnawong1 Date: Thu, 28 Nov 2024 11:12:14 +0000 Subject: [PATCH 11/13] remove iimport --- website/docs/reference/snapshot-configs.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 4d8d8e42d26..1867c6f12f0 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -7,8 +7,6 @@ meta: import ConfigResource from '/snippets/_config-description-resource.md'; import ConfigGeneral from '/snippets/_config-description-general.md'; -import SnapshotsYamlConfig from '/snippets/_snapshots-yaml-config.md'; - ## Related documentation * [Snapshots](/docs/build/snapshots) From 915e396b920e0d89ebe66fe08834f381da17078a Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Thu, 28 Nov 2024 11:41:55 +0000 Subject: [PATCH 12/13] Update snapshot-configs.md --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 1867c6f12f0..dfc2bd087f1 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -406,7 +406,7 @@ The following examples demonstrate how to configure snapshots using the `dbt_pro - You can also define some common configs in a snapshot's `config` block. We don't recommend this for a snapshot's required configuration, however. + You can also define some common configs in a snapshot's `config` block. However, we don't recommend this for a snapshot's required configuration. From 61d3792815ad46094c30293475811145a3483b21 Mon Sep 17 00:00:00 2001 From: Mirna Wong <89008547+mirnawong1@users.noreply.github.com> Date: Thu, 28 Nov 2024 12:03:50 +0000 Subject: [PATCH 13/13] Update snapshot-configs.md --- website/docs/reference/snapshot-configs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index dfc2bd087f1..4d9bcf8a049 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -119,7 +119,7 @@ snapshots: - +Starting from Versionless and Core v1.9, defining snapshots in a `.sql` file using a config block is a legacy method. You can define snapshots in YAML format using the latest [snapshot-specific configurations](/docs/build/snapshots#snapshot-meta-fields). For new snapshots, we recommend using these latest configs. If applying them to existing snapshots, you'll need to migrate over.