diff --git a/website/dbt-versions.js b/website/dbt-versions.js index 60efef64f75..d253829c0b7 100644 --- a/website/dbt-versions.js +++ b/website/dbt-versions.js @@ -42,6 +42,10 @@ exports.versions = [ * @property {string} lastVersion The last version the page is visible in the sidebar */ exports.versionedPages = [ + { + page: "reference/resource-configs/snapshot_meta_column_names", + firstVersion: "1.9", + }, { page: "reference/resource-configs/target_database", lastVersion: "1.8", diff --git a/website/docs/docs/build/snapshots.md b/website/docs/docs/build/snapshots.md index bcb65bd7810..c17350ab368 100644 --- a/website/docs/docs/build/snapshots.md +++ b/website/docs/docs/build/snapshots.md @@ -409,6 +409,8 @@ Basically – keep your query as simple as possible! Some reasonable exceptions Snapshot tables will be created as a clone of your source dataset, plus some additional meta-fields*. +Starting in 1.9 or with [dbt Cloud Versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless), these column names can be customized to your team or organizational conventions via the [`snapshot_meta_column_names`](/reference/resource-configs/snapshot_meta_column_names) config. + | Field | Meaning | Usage | | -------------- | ------- | ----- | | dbt_valid_from | The timestamp when this snapshot row was first inserted | This column can be used to order the different "versions" of a record. | diff --git a/website/docs/docs/dbt-versions/release-notes.md b/website/docs/docs/dbt-versions/release-notes.md index d8ac119deb8..2e23d394228 100644 --- a/website/docs/docs/dbt-versions/release-notes.md +++ b/website/docs/docs/dbt-versions/release-notes.md @@ -26,12 +26,12 @@ Release notes are grouped by month for both multi-tenant and virtual private clo - **Behavior change:** Set [`state_modified_compare_more_unrendered`](/reference/global-configs/behavior-changes#source-definitions-for-state) to true to reduce false positives for `state:modified` when configs differ between `dev` and `prod` environments. - **Behavior change:** Set the [`skip_nodes_if_on_run_start_fails`](/reference/global-configs/behavior-changes#failures-in-on-run-start-hooks) flag to `True` to skip all selected resources from running if there is a failure on an `on-run-start` hook. - **Enhancement**: In dbt Cloud Versionless, snapshots defined in SQL files can now use `config` defined in `schema.yml` YAML files. This update resolves the previous limitation that required snapshot properties to be defined exclusively in `dbt_project.yml` and/or a `config()` block within the SQL file. This enhancement will be included in the upcoming dbt Core v1.9 release. +- **New**: In dbt Cloud Versionless, the `snapshot_meta_column_names` config allows for customizing the snapshot metadata columns. This feature allows an organization to align these automatically-generated column names with their conventions, and will be included in the upcoming dbt Core 1.9 release. - **Enhancement**: In May 2024, dbt Cloud versionless began inferring a model's `primary_key` based on configured data tests and/or constraints within `manifest.json`. The inferred `primary_key` is visible in dbt Explorer and utilized by the dbt Cloud [compare changes](/docs/deploy/run-visibility#compare-tab) feature. This will also be released in dbt Core 1.9. Read about the [order dbt infers columns can be used as primary key of a model](https://github.com/dbt-labs/dbt-core/blob/7940ad5c7858ff11ef100260a372f2f06a86e71f/core/dbt/contracts/graph/nodes.py#L534-L541). - **New:** dbt Explorer now includes trust signal icons, which is currently available as a [Preview](/docs/dbt-versions/product-lifecycles#dbt-cloud). Trust signals offer a quick, at-a-glance view of data health when browsing your dbt models in Explorer. These icons indicate whether a model is **Healthy**, **Caution**, **Degraded**, or **Unknown**. For accurate health data, ensure the resource is up-to-date and has had a recent job run. Refer to [Trust signals](/docs/collaborate/explore-projects#trust-signals-for-resources) for more information. - **New:** Auto exposures are now available in Preview in dbt Cloud. Auto-exposures helps users understand how their models are used in downstream analytics tools to inform investments and reduce incidents. It imports and auto-generates exposures based on Tableau dashboards, with user-defined curation. To learn more, refer to [Auto exposures](/docs/collaborate/auto-exposures). - ## September 2024 - **New**: Use the new recommended syntax for [defining `foreign_key` constraints](/reference/resource-properties/constraints) using `refs`, available in dbt Cloud Versionless. This will soon be released in dbt Core v1.9. This new syntax will capture dependencies and works across different environments. diff --git a/website/docs/reference/resource-configs/snapshot_meta_column_names.md b/website/docs/reference/resource-configs/snapshot_meta_column_names.md new file mode 100644 index 00000000000..46aba7886d0 --- /dev/null +++ b/website/docs/reference/resource-configs/snapshot_meta_column_names.md @@ -0,0 +1,109 @@ +--- +resource_types: [snapshots] +description: "Snapshot meta column names" +datatype: "{}" +default_value: {"dbt_valid_from": "dbt_valid_from", "dbt_valid_to": "dbt_valid_to", "dbt_scd_id": "dbt_scd_id", "dbt_updated_at": "dbt_updated_at"} +id: "snapshot_meta_column_names" +--- + +Starting in 1.9 or with [versionless](/docs/dbt-versions/upgrade-dbt-version-in-cloud#versionless) dbt Cloud. + + + +```yaml +snapshots: + - name: + config: + snapshot_meta_column_names: + dbt_valid_from: + dbt_valid_to: + dbt_scd_id: + dbt_updated_at: + +``` + + + + + +```jinja2 +{{ + config( + snapshot_meta_column_names={ + "dbt_valid_from": "", + "dbt_valid_to": "", + "dbt_scd_id": "", + "dbt_updated_at": "", + } + ) +}} + +``` + + + + + +```yml +snapshots: + [](/reference/resource-configs/resource-path): + +snapshot_meta_column_names: + dbt_valid_from: + dbt_valid_to: + dbt_scd_id: + dbt_updated_at: + +``` + + + +## Description + +In order to align with an organization's naming conventions, the `snapshot_meta_column_names` config can be used to customize the names of the [metadata columns](/docs/build/snapshots#snapshot-meta-fields) within each snapshot. + +## Default + +By default, dbt snapshots use the following column names to track change history using [Type 2 slowly changing dimension](https://en.wikipedia.org/wiki/Slowly_changing_dimension#Type_2:_add_new_row) records: + +| Field | Meaning | Notes | +| -------------- | ------- | ----- | +| `dbt_valid_from` | The timestamp when this snapshot row was first inserted and became valid. | The value is affected by the [`strategy`](/reference/resource-configs/strategy). | +| `dbt_valid_to` | The timestamp when this row is no longer valid. | | +| `dbt_scd_id` | A unique key generated for each snapshot row. | This is used internally by dbt. | +| `dbt_updated_at` | The `updated_at` timestamp of the source record when this snapshot row was inserted. | This is used internally by dbt. | + +However, these column names can be customized using the `snapshot_meta_column_names` config. + +:::warning + +To avoid any unintentional data modification, dbt will **not** automatically apply any column renames. So if a user applies `snapshot_meta_column_names` config for a snapshot without updating the pre-existing table, they will get an error. We recommend either only using these settings for net-new snapshots, or arranging an update of pre-existing tables prior to committing a column name change. + +::: + +## Example + + + +```yaml +snapshots: + - name: orders_snapshot + relation: ref("orders") + config: + unique_key: id + strategy: check + check_cols: all + snapshot_meta_column_names: + dbt_valid_from: start_date + dbt_valid_to: end_date + dbt_scd_id: scd_id + dbt_updated_at: modified_date +``` + + + +The resulting snapshot table contains the configured meta column names: + +| id | scd_id | modified_date | start_date | end_date | +| -- | -------------------- | -------------------- | -------------------- | -------------------- | +| 1 | 60a1f1dbdf899a4dd... | 2024-10-02 ... | 2024-10-02 ... | 2024-10-02 ... | +| 2 | b1885d098f8bcff51... | 2024-10-02 ... | 2024-10-02 ... | | diff --git a/website/docs/reference/snapshot-configs.md b/website/docs/reference/snapshot-configs.md index 87063ce592e..e867747dc96 100644 --- a/website/docs/reference/snapshot-configs.md +++ b/website/docs/reference/snapshot-configs.md @@ -78,6 +78,7 @@ snapshots: [+](/reference/resource-configs/plus-prefix)[strategy](/reference/resource-configs/strategy): timestamp | check [+](/reference/resource-configs/plus-prefix)[updated_at](/reference/resource-configs/updated_at): [+](/reference/resource-configs/plus-prefix)[check_cols](/reference/resource-configs/check_cols): [] | all + [+](/reference/resource-configs/plus-prefix)[snapshot_meta_column_names](/reference/resource-configs/snapshot_meta_column_names): {} ``` @@ -111,6 +112,7 @@ snapshots: [strategy](/reference/resource-configs/strategy): timestamp | check [updated_at](/reference/resource-configs/updated_at): [check_cols](/reference/resource-configs/check_cols): [] | all + [snapshot_meta_column_names](/reference/resource-configs/snapshot_meta_column_names): {} ``` @@ -138,6 +140,7 @@ Configurations can be applied to snapshots using [YAML syntax](/docs/build/snaps [strategy](/reference/resource-configs/strategy)="timestamp" | "check", [updated_at](/reference/resource-configs/updated_at)="", [check_cols](/reference/resource-configs/check_cols)=[""] | "all" + [snapshot_meta_column_names](/reference/resource-configs/snapshot_meta_column_names)={} ) }} ``` diff --git a/website/sidebars.js b/website/sidebars.js index 771cde10562..540c6264b35 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -974,6 +974,7 @@ const sidebarSettings = { "reference/resource-configs/unique_key", "reference/resource-configs/updated_at", "reference/resource-configs/invalidate_hard_deletes", + "reference/resource-configs/snapshot_meta_column_names", ], }, {