diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml index bd8ede6a646f..12745c37a150 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/first_dbt_project.yml @@ -83,3 +83,8 @@ vars: multiple_column_names_conflicts_stg: test_normalization._airbyte_raw_multiple_column_names_conflicts multiple_column_names_conflicts_scd: test_normalization._airbyte_raw_multiple_column_names_conflicts multiple_column_names_conflicts: test_normalization._airbyte_raw_multiple_column_names_conflicts + types_testing_ab1: test_normalization._airbyte_raw_types_testing + types_testing_ab2: test_normalization._airbyte_raw_types_testing + types_testing_stg: test_normalization._airbyte_raw_types_testing + types_testing_scd: test_normalization._airbyte_raw_types_testing + types_testing: test_normalization._airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml index 97bf0d05cbd4..f51802427655 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/bigquery/test_simple_streams/models/generated/sources.yml @@ -13,3 +13,4 @@ sources: - name: _airbyte_raw_multiple_column_names_conflicts - name: _airbyte_raw_pos_dedup_cdcx - name: _airbyte_raw_renamed_dedup_cdc_excluded + - name: _airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml index 02cf2fd559fd..5d86e8cce453 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize model-paths: - - models +- models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests +- tests seed-paths: - - data +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs packages-install-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: true schema: false @@ -42,10 +42,10 @@ models: +tags: airbyte_internal_views +materialized: view dispatch: - - macro_namespace: dbt_utils - search_order: - - airbyte_utils - - dbt_utils +- macro_namespace: dbt_utils + search_order: + - airbyte_utils + - dbt_utils vars: json_column: _airbyte_data models_to_source: @@ -83,3 +83,8 @@ vars: multiple_column_names_conflicts_stg: test_normalization._airbyte_raw_multiple_column_names_conflicts multiple_column_names_conflicts_scd: test_normalization._airbyte_raw_multiple_column_names_conflicts multiple_column_names_conflicts: test_normalization._airbyte_raw_multiple_column_names_conflicts + types_testing_ab1: test_normalization._airbyte_raw_types_testing + types_testing_ab2: test_normalization._airbyte_raw_types_testing + types_testing_stg: test_normalization._airbyte_raw_types_testing + types_testing_scd: test_normalization._airbyte_raw_types_testing + types_testing: test_normalization._airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/sources.yml index 97bf0d05cbd4..f51802427655 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/clickhouse/test_simple_streams/models/generated/sources.yml @@ -13,3 +13,4 @@ sources: - name: _airbyte_raw_multiple_column_names_conflicts - name: _airbyte_raw_pos_dedup_cdcx - name: _airbyte_raw_renamed_dedup_cdc_excluded + - name: _airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml index 88b2a40e2d11..f187620c7c7c 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/dbt_project.yml @@ -79,6 +79,10 @@ vars: unnest_alias_ab2: test_normalization._airbyte_raw_unnest_alias unnest_alias_ab3: test_normalization._airbyte_raw_unnest_alias unnest_alias: test_normalization._airbyte_raw_unnest_alias + arrays_ab1: test_normalization._airbyte_raw_arrays + arrays_ab2: test_normalization._airbyte_raw_arrays + arrays_ab3: test_normalization._airbyte_raw_arrays + arrays: test_normalization._airbyte_raw_arrays nested_stream_with_co_2g_names_partition_ab1: test_normalization._airbyte_raw_nested_s__lting_into_long_names nested_stream_with_co_2g_names_partition_ab2: test_normalization._airbyte_raw_nested_s__lting_into_long_names nested_stream_with_co_2g_names_partition_ab3: test_normalization._airbyte_raw_nested_s__lting_into_long_names @@ -91,6 +95,10 @@ vars: unnest_alias_children_ab2: test_normalization._airbyte_raw_unnest_alias unnest_alias_children_ab3: test_normalization._airbyte_raw_unnest_alias unnest_alias_children: test_normalization._airbyte_raw_unnest_alias + arrays_nested_array_parent_ab1: test_normalization._airbyte_raw_arrays + arrays_nested_array_parent_ab2: test_normalization._airbyte_raw_arrays + arrays_nested_array_parent_ab3: test_normalization._airbyte_raw_arrays + arrays_nested_array_parent: test_normalization._airbyte_raw_arrays nested_stream_with_co_3double_array_data_ab1: test_normalization._airbyte_raw_nested_s__lting_into_long_names nested_stream_with_co_3double_array_data_ab2: test_normalization._airbyte_raw_nested_s__lting_into_long_names nested_stream_with_co_3double_array_data_ab3: test_normalization._airbyte_raw_nested_s__lting_into_long_names diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql index b5bce12e294f..e68283420cfd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -56,7 +56,7 @@ with numbers as ( select _airbyte_partition_hashid, json_value(_airbyte_nested_data, - '$."currency"') as currency, + '$."currency"' RETURNING CHAR) as currency, _airbyte_ab_id, _airbyte_emitted_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql index 279441127cad..4b276edcc316 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -56,7 +56,7 @@ with numbers as ( select _airbyte_partition_hashid, json_value(_airbyte_nested_data, - '$."id"') as id, + '$."id"' RETURNING CHAR) as id, _airbyte_ab_id, _airbyte_emitted_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/sources.yml index e34e562ac3fa..50def309c8c4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/models/generated/sources.yml @@ -6,6 +6,7 @@ sources: schema: false identifier: false tables: + - name: _airbyte_raw_arrays - name: _airbyte_raw_conflict_stream_array - name: _airbyte_raw_conflict_stream_name - name: _airbyte_raw_conflict_stream_scalar diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql index b5bce12e294f..e68283420cfd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co___names_partition_data.sql @@ -56,7 +56,7 @@ with numbers as ( select _airbyte_partition_hashid, json_value(_airbyte_nested_data, - '$."currency"') as currency, + '$."currency"' RETURNING CHAR) as currency, _airbyte_ab_id, _airbyte_emitted_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql index 279441127cad..4b276edcc316 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_co__ion_double_array_data.sql @@ -56,7 +56,7 @@ with numbers as ( select _airbyte_partition_hashid, json_value(_airbyte_nested_data, - '$."id"') as id, + '$."id"' RETURNING CHAR) as id, _airbyte_ab_id, _airbyte_emitted_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml index 5650b278a87c..bc7fa6d50166 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/dbt_project.yml @@ -79,3 +79,8 @@ vars: multiple_column_names_conflicts_stg: test_normalization._airbyte_raw_multiple_column_names_conflicts multiple_column_names_conflicts_scd: test_normalization._airbyte_raw_multiple_column_names_conflicts multiple_column_names_conflicts: test_normalization._airbyte_raw_multiple_column_names_conflicts + types_testing_ab1: test_normalization._airbyte_raw_types_testing + types_testing_ab2: test_normalization._airbyte_raw_types_testing + types_testing_stg: test_normalization._airbyte_raw_types_testing + types_testing_scd: test_normalization._airbyte_raw_types_testing + types_testing: test_normalization._airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index fd770070d75f..d9ea331937d2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -10,23 +10,23 @@ with __dbt__cte__exchange_rate_ab1 as ( -- depends_on: test_normalization._airbyte_raw_exchange_rate select json_value(_airbyte_data, - '$."id"') as id, + '$."id"' RETURNING CHAR) as id, json_value(_airbyte_data, - '$."currency"') as currency, + '$."currency"' RETURNING CHAR) as currency, json_value(_airbyte_data, - '$."date"') as `date`, + '$."date"' RETURNING CHAR) as `date`, json_value(_airbyte_data, - '$."timestamp_col"') as timestamp_col, + '$."timestamp_col"' RETURNING CHAR) as timestamp_col, json_value(_airbyte_data, - '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + '$."HKD@spéçiäl & characters"' RETURNING CHAR) as `HKD@spéçiäl & characters`, json_value(_airbyte_data, - '$."HKD_special___characters"') as hkd_special___characters, + '$."HKD_special___characters"' RETURNING CHAR) as hkd_special___characters, json_value(_airbyte_data, - '$."NZD"') as nzd, + '$."NZD"' RETURNING CHAR) as nzd, json_value(_airbyte_data, - '$."USD"') as usd, + '$."USD"' RETURNING CHAR) as usd, json_value(_airbyte_data, - '$."column___with__quotes"') as `column__'with"_quotes`, + '$."column___with__quotes"' RETURNING CHAR) as `column__'with"_quotes`, _airbyte_ab_id, _airbyte_emitted_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index c736f9f1ec46..367544ad79b7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -7,21 +7,21 @@ with __dbt__cte__dedup_exchange_rate_ab1 as ( -- depends_on: test_normalization._airbyte_raw_dedup_exchange_rate select json_value(_airbyte_data, - '$."id"') as id, + '$."id"' RETURNING CHAR) as id, json_value(_airbyte_data, - '$."currency"') as currency, + '$."currency"' RETURNING CHAR) as currency, json_value(_airbyte_data, - '$."date"') as `date`, + '$."date"' RETURNING CHAR) as `date`, json_value(_airbyte_data, - '$."timestamp_col"') as timestamp_col, + '$."timestamp_col"' RETURNING CHAR) as timestamp_col, json_value(_airbyte_data, - '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + '$."HKD@spéçiäl & characters"' RETURNING CHAR) as `HKD@spéçiäl & characters`, json_value(_airbyte_data, - '$."HKD_special___characters"') as hkd_special___characters, + '$."HKD_special___characters"' RETURNING CHAR) as hkd_special___characters, json_value(_airbyte_data, - '$."NZD"') as nzd, + '$."NZD"' RETURNING CHAR) as nzd, json_value(_airbyte_data, - '$."USD"') as usd, + '$."USD"' RETURNING CHAR) as usd, _airbyte_ab_id, _airbyte_emitted_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql index b9356dd1f6ba..1bd990b39925 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql @@ -7,19 +7,19 @@ with __dbt__cte__multiple_column_names_conflicts_ab1 as ( -- depends_on: test_normalization._airbyte_raw_multiple_column_names_conflicts select json_value(_airbyte_data, - '$."id"') as id, + '$."id"' RETURNING CHAR) as id, json_value(_airbyte_data, - '$."User Id"') as `User Id`, + '$."User Id"' RETURNING CHAR) as `User Id`, json_value(_airbyte_data, - '$."user_id"') as user_id, + '$."user_id"' RETURNING CHAR) as user_id, json_value(_airbyte_data, - '$."User id"') as `User id_1`, + '$."User id"' RETURNING CHAR) as `User id_1`, json_value(_airbyte_data, - '$."user id"') as `user id_2`, + '$."user id"' RETURNING CHAR) as `user id_2`, json_value(_airbyte_data, - '$."User@Id"') as `User@Id`, + '$."User@Id"' RETURNING CHAR) as `User@Id`, json_value(_airbyte_data, - '$."UserId"') as userid, + '$."UserId"' RETURNING CHAR) as userid, _airbyte_ab_id, _airbyte_emitted_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml index 97bf0d05cbd4..f51802427655 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/models/generated/sources.yml @@ -13,3 +13,4 @@ sources: - name: _airbyte_raw_multiple_column_names_conflicts - name: _airbyte_raw_pos_dedup_cdcx - name: _airbyte_raw_renamed_dedup_cdc_excluded + - name: _airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index fd770070d75f..d9ea331937d2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -10,23 +10,23 @@ with __dbt__cte__exchange_rate_ab1 as ( -- depends_on: test_normalization._airbyte_raw_exchange_rate select json_value(_airbyte_data, - '$."id"') as id, + '$."id"' RETURNING CHAR) as id, json_value(_airbyte_data, - '$."currency"') as currency, + '$."currency"' RETURNING CHAR) as currency, json_value(_airbyte_data, - '$."date"') as `date`, + '$."date"' RETURNING CHAR) as `date`, json_value(_airbyte_data, - '$."timestamp_col"') as timestamp_col, + '$."timestamp_col"' RETURNING CHAR) as timestamp_col, json_value(_airbyte_data, - '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + '$."HKD@spéçiäl & characters"' RETURNING CHAR) as `HKD@spéçiäl & characters`, json_value(_airbyte_data, - '$."HKD_special___characters"') as hkd_special___characters, + '$."HKD_special___characters"' RETURNING CHAR) as hkd_special___characters, json_value(_airbyte_data, - '$."NZD"') as nzd, + '$."NZD"' RETURNING CHAR) as nzd, json_value(_airbyte_data, - '$."USD"') as usd, + '$."USD"' RETURNING CHAR) as usd, json_value(_airbyte_data, - '$."column___with__quotes"') as `column__'with"_quotes`, + '$."column___with__quotes"' RETURNING CHAR) as `column__'with"_quotes`, _airbyte_ab_id, _airbyte_emitted_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index c736f9f1ec46..367544ad79b7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/mysql/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -7,21 +7,21 @@ with __dbt__cte__dedup_exchange_rate_ab1 as ( -- depends_on: test_normalization._airbyte_raw_dedup_exchange_rate select json_value(_airbyte_data, - '$."id"') as id, + '$."id"' RETURNING CHAR) as id, json_value(_airbyte_data, - '$."currency"') as currency, + '$."currency"' RETURNING CHAR) as currency, json_value(_airbyte_data, - '$."date"') as `date`, + '$."date"' RETURNING CHAR) as `date`, json_value(_airbyte_data, - '$."timestamp_col"') as timestamp_col, + '$."timestamp_col"' RETURNING CHAR) as timestamp_col, json_value(_airbyte_data, - '$."HKD@spéçiäl & characters"') as `HKD@spéçiäl & characters`, + '$."HKD@spéçiäl & characters"' RETURNING CHAR) as `HKD@spéçiäl & characters`, json_value(_airbyte_data, - '$."HKD_special___characters"') as hkd_special___characters, + '$."HKD_special___characters"' RETURNING CHAR) as hkd_special___characters, json_value(_airbyte_data, - '$."NZD"') as nzd, + '$."NZD"' RETURNING CHAR) as nzd, json_value(_airbyte_data, - '$."USD"') as usd, + '$."USD"' RETURNING CHAR) as usd, _airbyte_ab_id, _airbyte_emitted_at, diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml index a696787c00ab..e8b2b254e10b 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize source-paths: - - models +- models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests +- tests data-paths: - - data +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs modules-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: false schema: false @@ -42,7 +42,7 @@ models: +materialized: view vars: dbt_utils_dispatch_list: - - airbyte_utils + - airbyte_utils json_column: _airbyte_data models_to_source: exchange_rate_ab1: test_normalization.airbyte_raw_exchange_rate @@ -79,3 +79,8 @@ vars: multiple_column_names_conflicts_stg: test_normalization.airbyte_raw_multiple_column_names_conflicts multiple_column_names_conflicts_scd: test_normalization.airbyte_raw_multiple_column_names_conflicts multiple_column_names_conflicts: test_normalization.airbyte_raw_multiple_column_names_conflicts + types_testing_ab1: test_normalization.airbyte_raw_types_testing + types_testing_ab2: test_normalization.airbyte_raw_types_testing + types_testing_stg: test_normalization.airbyte_raw_types_testing + types_testing_scd: test_normalization.airbyte_raw_types_testing + types_testing: test_normalization.airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml index b5460fb4d43d..6fc61e6c97e1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/oracle/test_simple_streams/models/generated/sources.yml @@ -13,3 +13,4 @@ sources: - name: airbyte_raw_multiple_column_names_conflicts - name: airbyte_raw_pos_dedup_cdcx - name: airbyte_raw_renamed_dedup_cdc_excluded + - name: airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml index 77cd51053747..013a446b320a 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize model-paths: - - modified_models +- modified_models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests +- tests seed-paths: - - data +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs packages-install-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: true schema: false @@ -42,10 +42,10 @@ models: +tags: airbyte_internal_views +materialized: view dispatch: - - macro_namespace: dbt_utils - search_order: - - airbyte_utils - - dbt_utils +- macro_namespace: dbt_utils + search_order: + - airbyte_utils + - dbt_utils vars: json_column: _airbyte_data models_to_source: diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml index 200e87ca5ea7..12745c37a150 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize model-paths: - - models +- models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests +- tests seed-paths: - - data +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs packages-install-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: true schema: false @@ -42,10 +42,10 @@ models: +tags: airbyte_internal_views +materialized: view dispatch: - - macro_namespace: dbt_utils - search_order: - - airbyte_utils - - dbt_utils +- macro_namespace: dbt_utils + search_order: + - airbyte_utils + - dbt_utils vars: json_column: _airbyte_data models_to_source: @@ -83,3 +83,8 @@ vars: multiple_column_names_conflicts_stg: test_normalization._airbyte_raw_multiple_column_names_conflicts multiple_column_names_conflicts_scd: test_normalization._airbyte_raw_multiple_column_names_conflicts multiple_column_names_conflicts: test_normalization._airbyte_raw_multiple_column_names_conflicts + types_testing_ab1: test_normalization._airbyte_raw_types_testing + types_testing_ab2: test_normalization._airbyte_raw_types_testing + types_testing_stg: test_normalization._airbyte_raw_types_testing + types_testing_scd: test_normalization._airbyte_raw_types_testing + types_testing: test_normalization._airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/types_testing_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/types_testing_scd.sql new file mode 100644 index 000000000000..2f7da773390f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/types_testing_scd.sql @@ -0,0 +1,71 @@ + + + + create table "postgres".test_normalization."types_testing_scd" + as ( + +-- depends_on: ref('types_testing_stg') +with + +input_data as ( + select * + from "postgres"._airbyte_test_normalization."types_testing_stg" + -- types_testing from "postgres".test_normalization._airbyte_raw_types_testing +), + +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + md5(cast(coalesce(cast("id" as text), '') as text)) as _airbyte_unique_key, + "id", + big_integer, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by cast("id" as text) + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when row_number() over ( + partition by cast("id" as text) + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) = 1 then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_types_testing_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at + order by _airbyte_active_row desc, _airbyte_ab_id + ) as _airbyte_row_num, + md5(cast(coalesce(cast(_airbyte_unique_key as text), '') || '-' || coalesce(cast(_airbyte_start_at as text), '') || '-' || coalesce(cast(_airbyte_emitted_at as text), '') as text)) as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + "id", + big_integer, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_types_testing_hashid +from dedup_data where _airbyte_row_num = 1 + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/types_testing.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/types_testing.sql new file mode 100644 index 000000000000..e37aaa777d0b --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/types_testing.sql @@ -0,0 +1,23 @@ + + + + create table "postgres".test_normalization."types_testing" + as ( + +-- Final base SQL model +-- depends_on: "postgres".test_normalization."types_testing_scd" +select + _airbyte_unique_key, + "id", + big_integer, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at, + _airbyte_types_testing_hashid +from "postgres".test_normalization."types_testing_scd" +-- types_testing from "postgres".test_normalization._airbyte_raw_types_testing +where 1 = 1 +and _airbyte_active_row = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/types_testing_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/types_testing_stg.sql new file mode 100644 index 000000000000..536f2f307f5d --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/first_output/airbyte_incremental/test_normalization/types_testing_stg.sql @@ -0,0 +1,49 @@ + + + + create table "postgres"._airbyte_test_normalization."types_testing_stg" + as ( + +with __dbt__cte__types_testing_ab1 as ( + +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: "postgres".test_normalization._airbyte_raw_types_testing +select + jsonb_extract_path_text(_airbyte_data, 'id') as "id", + jsonb_extract_path_text(_airbyte_data, 'big_integer') as big_integer, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from "postgres".test_normalization._airbyte_raw_types_testing as table_alias +-- types_testing +where 1 = 1 + +), __dbt__cte__types_testing_ab2 as ( + +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: __dbt__cte__types_testing_ab1 +select + cast("id" as + int +) as "id", + cast(big_integer as + bigint +) as big_integer, + _airbyte_ab_id, + _airbyte_emitted_at, + now() as _airbyte_normalized_at +from __dbt__cte__types_testing_ab1 +-- types_testing +where 1 = 1 + +)-- SQL model to build a hash column based on the values of this record +-- depends_on: __dbt__cte__types_testing_ab2 +select + md5(cast(coalesce(cast("id" as text), '') || '-' || coalesce(cast(big_integer as text), '') as text)) as _airbyte_types_testing_hashid, + tmp.* +from __dbt__cte__types_testing_ab2 tmp +-- types_testing +where 1 = 1 + + ); + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/types_testing_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/types_testing_ab1.sql new file mode 100644 index 000000000000..f5676980d775 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/types_testing_ab1.sql @@ -0,0 +1,19 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema +-- depends_on: {{ source('test_normalization', '_airbyte_raw_types_testing') }} +select + {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as {{ adapter.quote('id') }}, + {{ json_extract_scalar('_airbyte_data', ['big_integer'], ['big_integer']) }} as big_integer, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ source('test_normalization', '_airbyte_raw_types_testing') }} as table_alias +-- types_testing +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at', this) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/types_testing_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/types_testing_ab2.sql new file mode 100644 index 000000000000..37ff75dc8e42 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_ctes/test_normalization/types_testing_ab2.sql @@ -0,0 +1,19 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to cast each column to its adequate SQL type converted from the JSON schema type +-- depends_on: {{ ref('types_testing_ab1') }} +select + cast({{ adapter.quote('id') }} as {{ dbt_utils.type_int() }}) as {{ adapter.quote('id') }}, + cast(big_integer as {{ dbt_utils.type_bigint() }}) as big_integer, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at +from {{ ref('types_testing_ab1') }} +-- types_testing +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at', this) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/types_testing_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/types_testing_scd.sql new file mode 100644 index 000000000000..b7799bb40dbe --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/types_testing_scd.sql @@ -0,0 +1,161 @@ +{{ config( + indexes = [{'columns':['_airbyte_active_row','_airbyte_unique_key_scd','_airbyte_emitted_at'],'type': 'btree'}], + unique_key = "_airbyte_unique_key_scd", + schema = "test_normalization", + post_hook = [" + {% + set final_table_relation = adapter.get_relation( + database=this.database, + schema=this.schema, + identifier='types_testing' + ) + %} + {# + If the final table doesn't exist, then obviously we can't delete anything from it. + Also, after a reset, the final table is created without the _airbyte_unique_key column (this column is created during the first sync) + So skip this deletion if the column doesn't exist. (in this case, the table is guaranteed to be empty anyway) + #} + {% + if final_table_relation is not none and '_airbyte_unique_key' in adapter.get_columns_in_relation(final_table_relation)|map(attribute='name') + %} + -- Delete records which are no longer active: + -- This query is equivalent, but the left join version is more performant: + -- delete from final_table where unique_key in ( + -- select unique_key from scd_table where 1 = 1 + -- ) and unique_key not in ( + -- select unique_key from scd_table where active_row = 1 + -- ) + -- We're incremental against normalized_at rather than emitted_at because we need to fetch the SCD + -- entries that were _updated_ recently. This is because a deleted record will have an SCD record + -- which was emitted a long time ago, but recently re-normalized to have active_row = 0. + delete from {{ final_table_relation }} where {{ final_table_relation }}._airbyte_unique_key in ( + select recent_records.unique_key + from ( + select distinct _airbyte_unique_key as unique_key + from {{ this }} + where 1=1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('types_testing')) }} + ) recent_records + left join ( + select _airbyte_unique_key as unique_key, count(_airbyte_unique_key) as active_count + from {{ this }} + where _airbyte_active_row = 1 {{ incremental_clause('_airbyte_normalized_at', this.schema + '.' + adapter.quote('types_testing')) }} + group by _airbyte_unique_key + ) active_counts + on recent_records.unique_key = active_counts.unique_key + where active_count is null or active_count = 0 + ) + {% else %} + -- We have to have a non-empty query, so just do a noop delete + delete from {{ this }} where 1=0 + {% endif %} + ","delete from _airbyte_test_normalization.types_testing_stg where _airbyte_emitted_at != (select max(_airbyte_emitted_at) from _airbyte_test_normalization.types_testing_stg)"], + tags = [ "top-level" ] +) }} +-- depends_on: ref('types_testing_stg') +with +{% if is_incremental() %} +new_data as ( + -- retrieve incremental "new" data + select + * + from {{ ref('types_testing_stg') }} + -- types_testing from {{ source('test_normalization', '_airbyte_raw_types_testing') }} + where 1 = 1 + {{ incremental_clause('_airbyte_emitted_at', this) }} +), +new_data_ids as ( + -- build a subset of _airbyte_unique_key from rows that are new + select distinct + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key + from new_data +), +empty_new_data as ( + -- build an empty table to only keep the table's column types + select * from new_data where 1 = 0 +), +previous_active_scd_data as ( + -- retrieve "incomplete old" data that needs to be updated with an end date because of new changes + select + {{ star_intersect(ref('types_testing_stg'), this, from_alias='inc_data', intersect_alias='this_data') }} + from {{ this }} as this_data + -- make a join with new_data using primary key to filter active data that need to be updated only + join new_data_ids on this_data._airbyte_unique_key = new_data_ids._airbyte_unique_key + -- force left join to NULL values (we just need to transfer column types only for the star_intersect macro on schema changes) + left join empty_new_data as inc_data on this_data._airbyte_ab_id = inc_data._airbyte_ab_id + where _airbyte_active_row = 1 +), +input_data as ( + select {{ dbt_utils.star(ref('types_testing_stg')) }} from new_data + union all + select {{ dbt_utils.star(ref('types_testing_stg')) }} from previous_active_scd_data +), +{% else %} +input_data as ( + select * + from {{ ref('types_testing_stg') }} + -- types_testing from {{ source('test_normalization', '_airbyte_raw_types_testing') }} +), +{% endif %} +scd_data as ( + -- SQL model to build a Type 2 Slowly Changing Dimension (SCD) table for each record identified by their primary key + select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + ]) }} as _airbyte_unique_key, + {{ adapter.quote('id') }}, + big_integer, + _airbyte_emitted_at as _airbyte_start_at, + lag(_airbyte_emitted_at) over ( + partition by cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) as _airbyte_end_at, + case when row_number() over ( + partition by cast({{ adapter.quote('id') }} as {{ dbt_utils.type_string() }}) + order by + _airbyte_emitted_at is null asc, + _airbyte_emitted_at desc, + _airbyte_emitted_at desc + ) = 1 then 1 else 0 end as _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + _airbyte_types_testing_hashid + from input_data +), +dedup_data as ( + select + -- we need to ensure de-duplicated rows for merge/update queries + -- additionally, we generate a unique key for the scd table + row_number() over ( + partition by + _airbyte_unique_key, + _airbyte_start_at, + _airbyte_emitted_at + order by _airbyte_active_row desc, _airbyte_ab_id + ) as _airbyte_row_num, + {{ dbt_utils.surrogate_key([ + '_airbyte_unique_key', + '_airbyte_start_at', + '_airbyte_emitted_at' + ]) }} as _airbyte_unique_key_scd, + scd_data.* + from scd_data +) +select + _airbyte_unique_key, + _airbyte_unique_key_scd, + {{ adapter.quote('id') }}, + big_integer, + _airbyte_start_at, + _airbyte_end_at, + _airbyte_active_row, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_types_testing_hashid +from dedup_data where _airbyte_row_num = 1 + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/types_testing.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/types_testing.sql new file mode 100644 index 000000000000..28237c8c416f --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/types_testing.sql @@ -0,0 +1,22 @@ +{{ config( + indexes = [{'columns':['_airbyte_unique_key'],'unique':True}], + unique_key = "_airbyte_unique_key", + schema = "test_normalization", + tags = [ "top-level" ] +) }} +-- Final base SQL model +-- depends_on: {{ ref('types_testing_scd') }} +select + _airbyte_unique_key, + {{ adapter.quote('id') }}, + big_integer, + _airbyte_ab_id, + _airbyte_emitted_at, + {{ current_timestamp() }} as _airbyte_normalized_at, + _airbyte_types_testing_hashid +from {{ ref('types_testing_scd') }} +-- types_testing from {{ source('test_normalization', '_airbyte_raw_types_testing') }} +where 1 = 1 +and _airbyte_active_row = 1 +{{ incremental_clause('_airbyte_emitted_at', this) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/types_testing_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/types_testing_stg.sql new file mode 100644 index 000000000000..941b939b2c8c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/airbyte_incremental/test_normalization/types_testing_stg.sql @@ -0,0 +1,19 @@ +{{ config( + indexes = [{'columns':['_airbyte_emitted_at'],'type':'btree'}], + unique_key = '_airbyte_ab_id', + schema = "_airbyte_test_normalization", + tags = [ "top-level-intermediate" ] +) }} +-- SQL model to build a hash column based on the values of this record +-- depends_on: {{ ref('types_testing_ab2') }} +select + {{ dbt_utils.surrogate_key([ + adapter.quote('id'), + 'big_integer', + ]) }} as _airbyte_types_testing_hashid, + tmp.* +from {{ ref('types_testing_ab2') }} tmp +-- types_testing +where 1 = 1 +{{ incremental_clause('_airbyte_emitted_at', this) }} + diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml index 97bf0d05cbd4..f51802427655 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/models/generated/sources.yml @@ -13,3 +13,4 @@ sources: - name: _airbyte_raw_multiple_column_names_conflicts - name: _airbyte_raw_pos_dedup_cdcx - name: _airbyte_raw_renamed_dedup_cdc_excluded + - name: _airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/sources.yml index 79ad1a1bb5c5..6a5d7bdc09a1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/modified_models/generated/sources.yml @@ -1,12 +1,12 @@ version: 2 sources: - - name: test_normalization - quoting: - database: true - schema: false - identifier: false - tables: - - name: _airbyte_raw_dedup_cdc_excluded - - name: _airbyte_raw_dedup_exchange_rate - - name: _airbyte_raw_exchange_rate - - name: _airbyte_raw_renamed_dedup_cdc_excluded +- name: test_normalization + quoting: + database: true + schema: false + identifier: false + tables: + - name: _airbyte_raw_dedup_cdc_excluded + - name: _airbyte_raw_dedup_exchange_rate + - name: _airbyte_raw_exchange_rate + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/types_testing_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/types_testing_scd.sql new file mode 100644 index 000000000000..371b6a534f73 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/types_testing_scd.sql @@ -0,0 +1,15 @@ + + + delete from "postgres".test_normalization."types_testing_scd" + where (_airbyte_unique_key_scd) in ( + select (_airbyte_unique_key_scd) + from "types_testing_scd__dbt_tmp" + ); + + + insert into "postgres".test_normalization."types_testing_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "big_integer", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_types_testing_hashid") + ( + select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "big_integer", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_types_testing_hashid" + from "types_testing_scd__dbt_tmp" + ) + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/types_testing.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/types_testing.sql new file mode 100644 index 000000000000..1839a672a866 --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/types_testing.sql @@ -0,0 +1,15 @@ + + + delete from "postgres".test_normalization."types_testing" + where (_airbyte_unique_key) in ( + select (_airbyte_unique_key) + from "types_testing__dbt_tmp" + ); + + + insert into "postgres".test_normalization."types_testing" ("_airbyte_unique_key", "id", "big_integer", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_types_testing_hashid") + ( + select "_airbyte_unique_key", "id", "big_integer", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_types_testing_hashid" + from "types_testing__dbt_tmp" + ) + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/types_testing_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/types_testing_stg.sql new file mode 100644 index 000000000000..1bf66940ba4c --- /dev/null +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/postgres/test_simple_streams/second_output/airbyte_incremental/test_normalization/types_testing_stg.sql @@ -0,0 +1,15 @@ + + + delete from "postgres"._airbyte_test_normalization."types_testing_stg" + where (_airbyte_ab_id) in ( + select (_airbyte_ab_id) + from "types_testing_stg__dbt_tmp" + ); + + + insert into "postgres"._airbyte_test_normalization."types_testing_stg" ("_airbyte_types_testing_hashid", "id", "big_integer", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at") + ( + select "_airbyte_types_testing_hashid", "id", "big_integer", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at" + from "types_testing_stg__dbt_tmp" + ) + \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml index 12f7ddeae970..777e4897e28c 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize model-paths: - - models +- models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests +- tests seed-paths: - - data +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs packages-install-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: true schema: false @@ -44,76 +44,84 @@ models: +tags: airbyte_internal_views +materialized: view dispatch: - - macro_namespace: dbt_utils - search_order: - - airbyte_utils - - dbt_utils +- macro_namespace: dbt_utils + search_order: + - airbyte_utils + - dbt_utils vars: json_column: _airbyte_data models_to_source: - nested_stream_with_complex_columns_resulting_into_long_names_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_stg: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_scd: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - non_nested_stream_without_namespace_resulting_into_long_names_ab1: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - non_nested_stream_without_namespace_resulting_into_long_names_ab2: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - non_nested_stream_without_namespace_resulting_into_long_names_ab3: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - non_nested_stream_without_namespace_resulting_into_long_names: test_normalization._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names - some_stream_that_was_empty_ab1: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_ab2: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_stg: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty_scd: test_normalization._airbyte_raw_some_stream_that_was_empty - some_stream_that_was_empty: test_normalization._airbyte_raw_some_stream_that_was_empty + nested_stream_with_complex_columns_resulting_into_long_names_ab1: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_ab2: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_stg: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_scd: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + non_nested_stream_without_namespace_resulting_into_long_names_ab1: test_normalization_kkvia._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names + non_nested_stream_without_namespace_resulting_into_long_names_ab2: test_normalization_kkvia._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names + non_nested_stream_without_namespace_resulting_into_long_names_ab3: test_normalization_kkvia._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names + non_nested_stream_without_namespace_resulting_into_long_names: test_normalization_kkvia._airbyte_raw_non_nested_stream_without_namespace_resulting_into_long_names + some_stream_that_was_empty_ab1: test_normalization_kkvia._airbyte_raw_some_stream_that_was_empty + some_stream_that_was_empty_ab2: test_normalization_kkvia._airbyte_raw_some_stream_that_was_empty + some_stream_that_was_empty_stg: test_normalization_kkvia._airbyte_raw_some_stream_that_was_empty + some_stream_that_was_empty_scd: test_normalization_kkvia._airbyte_raw_some_stream_that_was_empty + some_stream_that_was_empty: test_normalization_kkvia._airbyte_raw_some_stream_that_was_empty simple_stream_with_namespace_resulting_into_long_names_ab1: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names simple_stream_with_namespace_resulting_into_long_names_ab2: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names simple_stream_with_namespace_resulting_into_long_names_ab3: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names simple_stream_with_namespace_resulting_into_long_names: test_normalization_namespace._airbyte_raw_simple_stream_with_namespace_resulting_into_long_names - conflict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_scalar_ab1: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar_ab2: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar_ab3: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_scalar: test_normalization._airbyte_raw_conflict_stream_scalar - conflict_stream_array_ab1: test_normalization._airbyte_raw_conflict_stream_array - conflict_stream_array_ab2: test_normalization._airbyte_raw_conflict_stream_array - conflict_stream_array_ab3: test_normalization._airbyte_raw_conflict_stream_array - conflict_stream_array: test_normalization._airbyte_raw_conflict_stream_array - unnest_alias_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias: test_normalization._airbyte_raw_unnest_alias - nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - conflict_stream_name_conflict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name - unnest_alias_children_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children: test_normalization._airbyte_raw_unnest_alias - nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - nested_stream_with_complex_columns_resulting_into_long_names_partition_data: test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names - conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3: test_normalization._airbyte_raw_conflict_stream_name - conflict_stream_name_conflict_stream_name_conflict_stream_name: test_normalization._airbyte_raw_conflict_stream_name - unnest_alias_children_owner_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner_column___with__quotes_ab1: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner_column___with__quotes_ab2: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner_column___with__quotes_ab3: test_normalization._airbyte_raw_unnest_alias - unnest_alias_children_owner_column___with__quotes: test_normalization._airbyte_raw_unnest_alias + conflict_stream_name_ab1: test_normalization_kkvia._airbyte_raw_conflict_stream_name + conflict_stream_name_ab2: test_normalization_kkvia._airbyte_raw_conflict_stream_name + conflict_stream_name_ab3: test_normalization_kkvia._airbyte_raw_conflict_stream_name + conflict_stream_name: test_normalization_kkvia._airbyte_raw_conflict_stream_name + conflict_stream_scalar_ab1: test_normalization_kkvia._airbyte_raw_conflict_stream_scalar + conflict_stream_scalar_ab2: test_normalization_kkvia._airbyte_raw_conflict_stream_scalar + conflict_stream_scalar_ab3: test_normalization_kkvia._airbyte_raw_conflict_stream_scalar + conflict_stream_scalar: test_normalization_kkvia._airbyte_raw_conflict_stream_scalar + conflict_stream_array_ab1: test_normalization_kkvia._airbyte_raw_conflict_stream_array + conflict_stream_array_ab2: test_normalization_kkvia._airbyte_raw_conflict_stream_array + conflict_stream_array_ab3: test_normalization_kkvia._airbyte_raw_conflict_stream_array + conflict_stream_array: test_normalization_kkvia._airbyte_raw_conflict_stream_array + unnest_alias_ab1: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_ab2: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_ab3: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias: test_normalization_kkvia._airbyte_raw_unnest_alias + arrays_ab1: test_normalization_kkvia._airbyte_raw_arrays + arrays_ab2: test_normalization_kkvia._airbyte_raw_arrays + arrays_ab3: test_normalization_kkvia._airbyte_raw_arrays + arrays: test_normalization_kkvia._airbyte_raw_arrays + nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_ab2: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + conflict_stream_name_conflict_stream_name_ab1: test_normalization_kkvia._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name_ab2: test_normalization_kkvia._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name_ab3: test_normalization_kkvia._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name: test_normalization_kkvia._airbyte_raw_conflict_stream_name + unnest_alias_children_ab1: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_children_ab2: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_children_ab3: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_children: test_normalization_kkvia._airbyte_raw_unnest_alias + arrays_nested_array_parent_ab1: test_normalization_kkvia._airbyte_raw_arrays + arrays_nested_array_parent_ab2: test_normalization_kkvia._airbyte_raw_arrays + arrays_nested_array_parent_ab3: test_normalization_kkvia._airbyte_raw_arrays + arrays_nested_array_parent: test_normalization_kkvia._airbyte_raw_arrays + nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab2: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab2: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + nested_stream_with_complex_columns_resulting_into_long_names_partition_data: test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + conflict_stream_name_conflict_stream_name_conflict_stream_name_ab1: test_normalization_kkvia._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name_conflict_stream_name_ab2: test_normalization_kkvia._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name_conflict_stream_name_ab3: test_normalization_kkvia._airbyte_raw_conflict_stream_name + conflict_stream_name_conflict_stream_name_conflict_stream_name: test_normalization_kkvia._airbyte_raw_conflict_stream_name + unnest_alias_children_owner_ab1: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_children_owner_ab2: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_children_owner_ab3: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_children_owner: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_children_owner_column___with__quotes_ab1: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_children_owner_column___with__quotes_ab2: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_children_owner_column___with__quotes_ab3: test_normalization_kkvia._airbyte_raw_unnest_alias + unnest_alias_children_owner_column___with__quotes: test_normalization_kkvia._airbyte_raw_unnest_alias diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 753b62319771..a6ebefd75104 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -2,7 +2,7 @@ create table - "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names_scd__dbt_tmp" + "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_scd" compound sortkey(_airbyte_active_row,_airbyte_unique_key_scd,_airbyte_emitted_at) @@ -14,8 +14,8 @@ with input_data as ( select * - from "integrationtests"._airbyte_test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_stg" - -- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names + from "integrationtests"._airbyte_test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_stg" + -- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names ), scd_data as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql index 6961ef2198dc..d341fa23adba 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -2,7 +2,7 @@ create table - "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names__dbt_tmp" + "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names" compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) @@ -10,7 +10,7 @@ as ( -- Final base SQL model --- depends_on: "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" +-- depends_on: "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_scd" select _airbyte_unique_key, id, @@ -20,8 +20,8 @@ select _airbyte_emitted_at, getdate() as _airbyte_normalized_at, _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" --- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names +from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_scd" +-- nested_stream_with_complex_columns_resulting_into_long_names from "integrationtests".test_normalization_kkvia._airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names where 1 = 1 and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index 8348fdeb8132..f4e614066bc5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -2,7 +2,7 @@ create table - "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names_partition__dbt_tmp" + "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition" compound sortkey(_airbyte_emitted_at) @@ -12,7 +12,7 @@ with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" +-- depends_on: "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_scd" select _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid, "partition"."double_array_data" as double_array_data, @@ -20,7 +20,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" as table_alias +from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_scd" as table_alias -- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition where 1 = 1 and "partition" is not null @@ -62,7 +62,7 @@ select getdate() as _airbyte_normalized_at, _airbyte_partition_hashid from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_ab3 --- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" +-- partition at nested_stream_with_complex_columns_resulting_into_long_names/partition from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_scd" where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql index 3f46c9e431c0..a44c74cf8b61 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -2,7 +2,7 @@ create table - "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names_partition_data__dbt_tmp" + "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" compound sortkey(_airbyte_emitted_at) @@ -12,13 +12,13 @@ with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +-- depends_on: "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition" with joined as ( select table_alias._airbyte_partition_hashid as _airbyte_hashid, _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.data as _airbyte_nested_data + from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.data as _airbyte_nested_data ) select _airbyte_partition_hashid, @@ -26,7 +26,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias -- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA left join joined on _airbyte_partition_hashid = joined._airbyte_hashid where 1 = 1 @@ -67,7 +67,7 @@ select getdate() as _airbyte_normalized_at, _airbyte_data_hashid from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab3 --- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +-- data at nested_stream_with_complex_columns_resulting_into_long_names/partition/DATA from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition" where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index 84fbebb03b50..9c983bee824a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/first_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -2,7 +2,7 @@ create table - "integrationtests"."test_normalization"."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data__dbt_tmp" + "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" compound sortkey(_airbyte_emitted_at) @@ -12,13 +12,13 @@ with __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +-- depends_on: "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition" with joined as ( select table_alias._airbyte_partition_hashid as _airbyte_hashid, _airbyte_nested_data - from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.double_array_data as _airbyte_nested_data + from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias, table_alias.double_array_data as _airbyte_nested_data ) select _airbyte_partition_hashid, @@ -26,7 +26,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias +from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition" as table_alias -- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data left join joined on _airbyte_partition_hashid = joined._airbyte_hashid where 1 = 1 @@ -67,7 +67,7 @@ select getdate() as _airbyte_normalized_at, _airbyte_double_array_data_hashid from __dbt__cte__nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab3 --- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" +-- double_array_data at nested_stream_with_complex_columns_resulting_into_long_names/partition/double_array_data from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition" where 1 = 1 ); diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql index c6c4c7bb3973..dc6e73a21b2a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab1.sql @@ -1,11 +1,11 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_kkvia", tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +-- depends_on: {{ source('test_normalization_kkvia', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['date'], ['date']) }} as date, @@ -13,7 +13,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias +from {{ source('test_normalization_kkvia', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} as table_alias -- nested_stream_with_complex_columns_resulting_into_long_names where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql index 41c1f86de0c7..36adf1f5b290 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_ab2.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_kkvia", tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql index 7d9968fe7170..544efffb78bb 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_ab1.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_kkvia", tags = [ "nested-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql index b3f16a06f544..fd174c632f1e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data_ab1.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_kkvia", tags = [ "nested-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql index 7fe25a4c8eca..3149171ebd94 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_ctes/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data_ab1.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_kkvia", tags = [ "nested-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index 627f56e3ad2a..023ccabd8183 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_active_row", "_airbyte_unique_key_scd", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", + schema = "test_normalization_kkvia", post_hook = [" {% set final_table_relation = adapter.get_relation( @@ -48,7 +48,7 @@ -- We have to have a non-empty query, so just do a noop delete delete from {{ this }} where 1=0 {% endif %} - ","drop view _airbyte_test_normalization.nested_stream_with_complex_columns_resulting_into_long_names_stg"], + ","drop view _airbyte_test_normalization_kkvia.nested_stream_with_complex_columns_resulting_into_long_names_stg"], tags = [ "top-level" ] ) }} -- depends_on: ref('nested_stream_with_complex_columns_resulting_into_long_names_stg') @@ -59,7 +59,7 @@ new_data as ( select * from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_stg') }} - -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_kkvia', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} ), @@ -95,7 +95,7 @@ input_data as ( input_data as ( select * from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_stg') }} - -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} + -- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_kkvia', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} ), {% endif %} scd_data as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql index 7b608604bbee..fc33e96cc22a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key", - schema = "test_normalization", + schema = "test_normalization_kkvia", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -16,7 +16,7 @@ select {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid from {{ ref('nested_stream_with_complex_columns_resulting_into_long_names_scd') }} --- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} +-- nested_stream_with_complex_columns_resulting_into_long_names from {{ source('test_normalization_kkvia', '_airbyte_raw_nested_stream_with_complex_columns_resulting_into_long_names') }} where 1 = 1 and _airbyte_active_row = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index 10a422df08a2..de507697b4b3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "test_normalization", + schema = "test_normalization_kkvia", tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql index 4551ff86747c..72e45a0bcc1f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "test_normalization", + schema = "test_normalization_kkvia", tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index 3f7bbf34e3e8..1df74f4251a2 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -1,6 +1,6 @@ {{ config( sort = "_airbyte_emitted_at", - schema = "test_normalization", + schema = "test_normalization_kkvia", tags = [ "nested" ] ) }} -- Final base SQL model diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/sources.yml index 92fa4c9a2580..807ff573cdf3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/models/generated/sources.yml @@ -1,11 +1,12 @@ version: 2 sources: -- name: test_normalization +- name: test_normalization_kkvia quoting: database: true schema: false identifier: false tables: + - name: _airbyte_raw_arrays - name: _airbyte_raw_conflict_stream_array - name: _airbyte_raw_conflict_stream_name - name: _airbyte_raw_conflict_stream_scalar diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql index a3986d744fa8..72a8d2d28dd5 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/scd/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_scd.sql @@ -1,13 +1,13 @@ - delete from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" + delete from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_scd" where (_airbyte_unique_key_scd) in ( select (_airbyte_unique_key_scd) from "nested_stream_with_complex_columns_resulti__dbt_tmp" ); - insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") + insert into "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") ( select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "date", "partition", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" from "nested_stream_with_complex_columns_resulti__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql index 842e9a979c68..d9ccdfbcddf8 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names.sql @@ -1,13 +1,13 @@ - delete from "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" + delete from "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names" where (_airbyte_unique_key) in ( select (_airbyte_unique_key) from "nested_stream_with_complex_columns_resulti__dbt_tmp" ); - insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") + insert into "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names" ("_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid") ( select "_airbyte_unique_key", "id", "date", "partition", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid" from "nested_stream_with_complex_columns_resulti__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql index 9bf3ef8552d7..01d98fbb098f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition.sql @@ -1,7 +1,7 @@ - insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition" ("_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid") + insert into "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition" ("_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid") ( select "_airbyte_nested_stream_with_complex_columns_resulting_into_long_names_hashid", "double_array_data", "data", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_partition_hashid" from "nested_stream_with_complex_columns_resulti__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql index ec83ef6267a0..81234ba37787 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_data.sql @@ -1,7 +1,7 @@ - insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" ("_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid") + insert into "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition_data" ("_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid") ( select "_airbyte_partition_hashid", "currency", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_data_hashid" from "nested_stream_with_complex_columns_resulti__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql index 05efdbcc1ce2..6c5bc9659782 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_nested_streams/second_output/airbyte_incremental/test_normalization/nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data.sql @@ -1,7 +1,7 @@ - insert into "integrationtests".test_normalization."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" ("_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid") + insert into "integrationtests".test_normalization_kkvia."nested_stream_with_complex_columns_resulting_into_long_names_partition_double_array_data" ("_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid") ( select "_airbyte_partition_hashid", "id", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_double_array_data_hashid" from "nested_stream_with_complex_columns_resulti__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml index 06d2109d3356..7483d99da213 100755 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize model-paths: - - modified_models +- modified_models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests +- tests seed-paths: - - data +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs packages-install-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: true schema: false @@ -44,29 +44,29 @@ models: +tags: airbyte_internal_views +materialized: view dispatch: - - macro_namespace: dbt_utils - search_order: - - airbyte_utils - - dbt_utils +- macro_namespace: dbt_utils + search_order: + - airbyte_utils + - dbt_utils vars: json_column: _airbyte_data models_to_source: - exchange_rate_ab1: test_normalization._airbyte_raw_exchange_rate - exchange_rate_ab2: test_normalization._airbyte_raw_exchange_rate - exchange_rate_ab3: test_normalization._airbyte_raw_exchange_rate - exchange_rate: test_normalization._airbyte_raw_exchange_rate - dedup_exchange_rate_ab1: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_ab2: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_stg: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_scd: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate: test_normalization._airbyte_raw_dedup_exchange_rate - renamed_dedup_cdc_excluded_ab1: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_ab2: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_stg: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_scd: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - dedup_cdc_excluded_ab1: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_ab2: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_stg: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_scd: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded: test_normalization._airbyte_raw_dedup_cdc_excluded + exchange_rate_ab1: test_normalization_iakoz._airbyte_raw_exchange_rate + exchange_rate_ab2: test_normalization_iakoz._airbyte_raw_exchange_rate + exchange_rate_ab3: test_normalization_iakoz._airbyte_raw_exchange_rate + exchange_rate: test_normalization_iakoz._airbyte_raw_exchange_rate + dedup_exchange_rate_ab1: test_normalization_iakoz._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_ab2: test_normalization_iakoz._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_stg: test_normalization_iakoz._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_scd: test_normalization_iakoz._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate: test_normalization_iakoz._airbyte_raw_dedup_exchange_rate + renamed_dedup_cdc_excluded_ab1: test_normalization_iakoz._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_ab2: test_normalization_iakoz._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_stg: test_normalization_iakoz._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_scd: test_normalization_iakoz._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded: test_normalization_iakoz._airbyte_raw_renamed_dedup_cdc_excluded + dedup_cdc_excluded_ab1: test_normalization_iakoz._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_ab2: test_normalization_iakoz._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_stg: test_normalization_iakoz._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_scd: test_normalization_iakoz._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded: test_normalization_iakoz._airbyte_raw_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml index 5b2760dc9d0f..616838c27347 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize model-paths: - - models +- models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests +- tests seed-paths: - - data +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs packages-install-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: true schema: false @@ -44,44 +44,49 @@ models: +tags: airbyte_internal_views +materialized: view dispatch: - - macro_namespace: dbt_utils - search_order: - - airbyte_utils - - dbt_utils +- macro_namespace: dbt_utils + search_order: + - airbyte_utils + - dbt_utils vars: json_column: _airbyte_data models_to_source: - exchange_rate_ab1: test_normalization._airbyte_raw_exchange_rate - exchange_rate_ab2: test_normalization._airbyte_raw_exchange_rate - exchange_rate_ab3: test_normalization._airbyte_raw_exchange_rate - exchange_rate: test_normalization._airbyte_raw_exchange_rate - dedup_exchange_rate_ab1: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_ab2: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_stg: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate_scd: test_normalization._airbyte_raw_dedup_exchange_rate - dedup_exchange_rate: test_normalization._airbyte_raw_dedup_exchange_rate - renamed_dedup_cdc_excluded_ab1: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_ab2: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_stg: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded_scd: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - renamed_dedup_cdc_excluded: test_normalization._airbyte_raw_renamed_dedup_cdc_excluded - dedup_cdc_excluded_ab1: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_ab2: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_stg: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded_scd: test_normalization._airbyte_raw_dedup_cdc_excluded - dedup_cdc_excluded: test_normalization._airbyte_raw_dedup_cdc_excluded - pos_dedup_cdcx_ab1: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_ab2: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_stg: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx_scd: test_normalization._airbyte_raw_pos_dedup_cdcx - pos_dedup_cdcx: test_normalization._airbyte_raw_pos_dedup_cdcx - 1_prefix_startwith_number_ab1: test_normalization._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number_ab2: test_normalization._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number_stg: test_normalization._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number_scd: test_normalization._airbyte_raw_1_prefix_startwith_number - 1_prefix_startwith_number: test_normalization._airbyte_raw_1_prefix_startwith_number - multiple_column_names_conflicts_ab1: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_ab2: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_stg: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts_scd: test_normalization._airbyte_raw_multiple_column_names_conflicts - multiple_column_names_conflicts: test_normalization._airbyte_raw_multiple_column_names_conflicts + exchange_rate_ab1: test_normalization_iakoz._airbyte_raw_exchange_rate + exchange_rate_ab2: test_normalization_iakoz._airbyte_raw_exchange_rate + exchange_rate_ab3: test_normalization_iakoz._airbyte_raw_exchange_rate + exchange_rate: test_normalization_iakoz._airbyte_raw_exchange_rate + dedup_exchange_rate_ab1: test_normalization_iakoz._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_ab2: test_normalization_iakoz._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_stg: test_normalization_iakoz._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate_scd: test_normalization_iakoz._airbyte_raw_dedup_exchange_rate + dedup_exchange_rate: test_normalization_iakoz._airbyte_raw_dedup_exchange_rate + renamed_dedup_cdc_excluded_ab1: test_normalization_iakoz._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_ab2: test_normalization_iakoz._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_stg: test_normalization_iakoz._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded_scd: test_normalization_iakoz._airbyte_raw_renamed_dedup_cdc_excluded + renamed_dedup_cdc_excluded: test_normalization_iakoz._airbyte_raw_renamed_dedup_cdc_excluded + dedup_cdc_excluded_ab1: test_normalization_iakoz._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_ab2: test_normalization_iakoz._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_stg: test_normalization_iakoz._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded_scd: test_normalization_iakoz._airbyte_raw_dedup_cdc_excluded + dedup_cdc_excluded: test_normalization_iakoz._airbyte_raw_dedup_cdc_excluded + pos_dedup_cdcx_ab1: test_normalization_iakoz._airbyte_raw_pos_dedup_cdcx + pos_dedup_cdcx_ab2: test_normalization_iakoz._airbyte_raw_pos_dedup_cdcx + pos_dedup_cdcx_stg: test_normalization_iakoz._airbyte_raw_pos_dedup_cdcx + pos_dedup_cdcx_scd: test_normalization_iakoz._airbyte_raw_pos_dedup_cdcx + pos_dedup_cdcx: test_normalization_iakoz._airbyte_raw_pos_dedup_cdcx + 1_prefix_startwith_number_ab1: test_normalization_iakoz._airbyte_raw_1_prefix_startwith_number + 1_prefix_startwith_number_ab2: test_normalization_iakoz._airbyte_raw_1_prefix_startwith_number + 1_prefix_startwith_number_stg: test_normalization_iakoz._airbyte_raw_1_prefix_startwith_number + 1_prefix_startwith_number_scd: test_normalization_iakoz._airbyte_raw_1_prefix_startwith_number + 1_prefix_startwith_number: test_normalization_iakoz._airbyte_raw_1_prefix_startwith_number + multiple_column_names_conflicts_ab1: test_normalization_iakoz._airbyte_raw_multiple_column_names_conflicts + multiple_column_names_conflicts_ab2: test_normalization_iakoz._airbyte_raw_multiple_column_names_conflicts + multiple_column_names_conflicts_stg: test_normalization_iakoz._airbyte_raw_multiple_column_names_conflicts + multiple_column_names_conflicts_scd: test_normalization_iakoz._airbyte_raw_multiple_column_names_conflicts + multiple_column_names_conflicts: test_normalization_iakoz._airbyte_raw_multiple_column_names_conflicts + types_testing_ab1: test_normalization_iakoz._airbyte_raw_types_testing + types_testing_ab2: test_normalization_iakoz._airbyte_raw_types_testing + types_testing_stg: test_normalization_iakoz._airbyte_raw_types_testing + types_testing_scd: test_normalization_iakoz._airbyte_raw_types_testing + types_testing: test_normalization_iakoz._airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 2d58288b13ea..70b5a026748b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -2,7 +2,7 @@ create table - "integrationtests"."test_normalization"."dedup_exchange_rate_scd__dbt_tmp" + "integrationtests".test_normalization_iakoz."dedup_exchange_rate_scd" compound sortkey(_airbyte_active_row,_airbyte_unique_key_scd,_airbyte_emitted_at) @@ -14,8 +14,8 @@ with input_data as ( select * - from "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_stg" - -- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate + from "integrationtests"._airbyte_test_normalization_iakoz."dedup_exchange_rate_stg" + -- dedup_exchange_rate from "integrationtests".test_normalization_iakoz._airbyte_raw_dedup_exchange_rate ), scd_data as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index b7670479fa0c..f4956a460edd 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -2,7 +2,7 @@ create table - "integrationtests"."test_normalization"."dedup_exchange_rate__dbt_tmp" + "integrationtests".test_normalization_iakoz."dedup_exchange_rate" compound sortkey(_airbyte_unique_key,_airbyte_emitted_at) @@ -10,7 +10,7 @@ as ( -- Final base SQL model --- depends_on: "integrationtests".test_normalization."dedup_exchange_rate_scd" +-- depends_on: "integrationtests".test_normalization_iakoz."dedup_exchange_rate_scd" select _airbyte_unique_key, id, @@ -25,8 +25,8 @@ select _airbyte_emitted_at, getdate() as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid -from "integrationtests".test_normalization."dedup_exchange_rate_scd" --- dedup_exchange_rate from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate +from "integrationtests".test_normalization_iakoz."dedup_exchange_rate_scd" +-- dedup_exchange_rate from "integrationtests".test_normalization_iakoz._airbyte_raw_dedup_exchange_rate where 1 = 1 and _airbyte_active_row = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql index f2537f70055b..8f109df3f8a6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,7 +1,7 @@ create table - "integrationtests".test_normalization."exchange_rate__dbt_tmp" + "integrationtests".test_normalization_iakoz."exchange_rate__dbt_tmp" compound sortkey(_airbyte_emitted_at) @@ -11,7 +11,7 @@ with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "integrationtests".test_normalization._airbyte_raw_exchange_rate +-- depends_on: "integrationtests".test_normalization_iakoz._airbyte_raw_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -25,7 +25,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_exchange_rate as table_alias +from "integrationtests".test_normalization_iakoz._airbyte_raw_exchange_rate as table_alias -- exchange_rate where 1 = 1 ), __dbt__cte__exchange_rate_ab2 as ( @@ -87,6 +87,6 @@ select getdate() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "integrationtests".test_normalization._airbyte_raw_exchange_rate +-- exchange_rate from "integrationtests".test_normalization_iakoz._airbyte_raw_exchange_rate where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index f20d8dbdc37b..dafa7b5500a9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,11 +1,11 @@ - create view "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_stg__dbt_tmp" as ( + create view "integrationtests"._airbyte_test_normalization_iakoz."dedup_exchange_rate_stg__dbt_tmp" as ( with __dbt__cte__dedup_exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate +-- depends_on: "integrationtests".test_normalization_iakoz._airbyte_raw_dedup_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -18,7 +18,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +from "integrationtests".test_normalization_iakoz._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql index 37786e8c5256..1d463363c5ee 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/first_output/airbyte_views/test_normalization/multiple_column_names_conflicts_stg.sql @@ -1,11 +1,11 @@ - create view "integrationtests"._airbyte_test_normalization."multiple_column_names_conflicts_stg__dbt_tmp" as ( + create view "integrationtests"._airbyte_test_normalization_iakoz."multiple_column_names_conflicts_stg__dbt_tmp" as ( with __dbt__cte__multiple_column_names_conflicts_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "integrationtests".test_normalization._airbyte_raw_multiple_column_names_conflicts +-- depends_on: "integrationtests".test_normalization_iakoz._airbyte_raw_multiple_column_names_conflicts select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."User Id" != '' then _airbyte_data."User Id" end as "user id", @@ -17,7 +17,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_multiple_column_names_conflicts as table_alias +from "integrationtests".test_normalization_iakoz._airbyte_raw_multiple_column_names_conflicts as table_alias -- multiple_column_names_conflicts where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index 17c4a88a2059..dc2d98ffed18 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,11 +1,11 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_iakoz", tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +-- depends_on: {{ source('test_normalization_iakoz', '_airbyte_raw_dedup_exchange_rate') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, @@ -18,7 +18,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias +from {{ source('test_normalization_iakoz', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index 796d4205f5ae..bc1ba799c7ec 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_iakoz", tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 683191e161c5..991883ed08c1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_active_row", "_airbyte_unique_key_scd", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", + schema = "test_normalization_iakoz", post_hook = [" {% set final_table_relation = adapter.get_relation( @@ -48,7 +48,7 @@ -- We have to have a non-empty query, so just do a noop delete delete from {{ this }} where 1=0 {% endif %} - ","drop view _airbyte_test_normalization.dedup_exchange_rate_stg"], + ","drop view _airbyte_test_normalization_iakoz.dedup_exchange_rate_stg"], tags = [ "top-level" ] ) }} -- depends_on: ref('dedup_exchange_rate_stg') @@ -59,7 +59,7 @@ new_data as ( select * from {{ ref('dedup_exchange_rate_stg') }} - -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + -- dedup_exchange_rate from {{ source('test_normalization_iakoz', '_airbyte_raw_dedup_exchange_rate') }} where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} ), @@ -97,7 +97,7 @@ input_data as ( input_data as ( select * from {{ ref('dedup_exchange_rate_stg') }} - -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + -- dedup_exchange_rate from {{ source('test_normalization_iakoz', '_airbyte_raw_dedup_exchange_rate') }} ), {% endif %} scd_data as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index d8b57a81b7cf..6258736d53b7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key", - schema = "test_normalization", + schema = "test_normalization_iakoz", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -21,7 +21,7 @@ select {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +-- dedup_exchange_rate from {{ source('test_normalization_iakoz', '_airbyte_raw_dedup_exchange_rate') }} where 1 = 1 and _airbyte_active_row = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql index dcf6f501bc35..d7e66dc983d1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "test_normalization", + schema = "test_normalization_iakoz", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -21,6 +21,6 @@ select {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +-- exchange_rate from {{ source('test_normalization_iakoz', '_airbyte_raw_exchange_rate') }} where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 8de81a6690f8..12d70e14095b 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_iakoz", tags = [ "top-level-intermediate" ] ) }} -- SQL model to build a hash column based on the values of this record diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml index 97bf0d05cbd4..725e4f3e041d 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/models/generated/sources.yml @@ -1,6 +1,6 @@ version: 2 sources: -- name: test_normalization +- name: test_normalization_iakoz quoting: database: true schema: false @@ -13,3 +13,4 @@ sources: - name: _airbyte_raw_multiple_column_names_conflicts - name: _airbyte_raw_pos_dedup_cdcx - name: _airbyte_raw_renamed_dedup_cdc_excluded + - name: _airbyte_raw_types_testing diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql index eca4c17d59fb..71f55e2758c7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab1.sql @@ -1,11 +1,11 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_iakoz", tags = [ "top-level-intermediate" ] ) }} -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +-- depends_on: {{ source('test_normalization_iakoz', '_airbyte_raw_dedup_exchange_rate') }} select {{ json_extract_scalar('_airbyte_data', ['id'], ['id']) }} as id, {{ json_extract_scalar('_airbyte_data', ['currency'], ['currency']) }} as currency, @@ -18,7 +18,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, {{ current_timestamp() }} as _airbyte_normalized_at -from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} as table_alias +from {{ source('test_normalization_iakoz', '_airbyte_raw_dedup_exchange_rate') }} as table_alias -- dedup_exchange_rate where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql index bf26dc2829f0..7a34e344d298 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_ctes/test_normalization/dedup_exchange_rate_ab2.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_iakoz", tags = [ "top-level-intermediate" ] ) }} -- SQL model to cast each column to its adequate SQL type converted from the JSON schema type diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 2582b1213c70..a530c5a2b9b7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_active_row", "_airbyte_unique_key_scd", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key_scd", - schema = "test_normalization", + schema = "test_normalization_iakoz", post_hook = [" {% set final_table_relation = adapter.get_relation( @@ -48,7 +48,7 @@ -- We have to have a non-empty query, so just do a noop delete delete from {{ this }} where 1=0 {% endif %} - ","drop view _airbyte_test_normalization.dedup_exchange_rate_stg"], + ","drop view _airbyte_test_normalization_iakoz.dedup_exchange_rate_stg"], tags = [ "top-level" ] ) }} -- depends_on: ref('dedup_exchange_rate_stg') @@ -59,7 +59,7 @@ new_data as ( select * from {{ ref('dedup_exchange_rate_stg') }} - -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + -- dedup_exchange_rate from {{ source('test_normalization_iakoz', '_airbyte_raw_dedup_exchange_rate') }} where 1 = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} ), @@ -97,7 +97,7 @@ input_data as ( input_data as ( select * from {{ ref('dedup_exchange_rate_stg') }} - -- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} + -- dedup_exchange_rate from {{ source('test_normalization_iakoz', '_airbyte_raw_dedup_exchange_rate') }} ), {% endif %} scd_data as ( diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 421177e81179..a1b9d4a5e118 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,7 +1,7 @@ {{ config( sort = ["_airbyte_unique_key", "_airbyte_emitted_at"], unique_key = "_airbyte_unique_key", - schema = "test_normalization", + schema = "test_normalization_iakoz", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -21,7 +21,7 @@ select {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_dedup_exchange_rate_hashid from {{ ref('dedup_exchange_rate_scd') }} --- dedup_exchange_rate from {{ source('test_normalization', '_airbyte_raw_dedup_exchange_rate') }} +-- dedup_exchange_rate from {{ source('test_normalization_iakoz', '_airbyte_raw_dedup_exchange_rate') }} where 1 = 1 and _airbyte_active_row = 1 {{ incremental_clause('_airbyte_emitted_at', this) }} diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql index e50ad0ad8a92..357e350dff1e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "test_normalization", + schema = "test_normalization_iakoz", tags = [ "top-level" ] ) }} -- Final base SQL model @@ -21,6 +21,6 @@ select {{ current_timestamp() }} as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from {{ ref('exchange_rate_ab3') }} --- exchange_rate from {{ source('test_normalization', '_airbyte_raw_exchange_rate') }} +-- exchange_rate from {{ source('test_normalization_iakoz', '_airbyte_raw_exchange_rate') }} where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index 59153246fdb5..11d30852f8a7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,7 +1,7 @@ {{ config( sort = "_airbyte_emitted_at", unique_key = '_airbyte_ab_id', - schema = "_airbyte_test_normalization", + schema = "_airbyte_test_normalization_iakoz", tags = [ "top-level-intermediate" ] ) }} -- SQL model to build a hash column based on the values of this record diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml index 79ad1a1bb5c5..dfa9910a894e 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/modified_models/generated/sources.yml @@ -1,12 +1,12 @@ version: 2 sources: - - name: test_normalization - quoting: - database: true - schema: false - identifier: false - tables: - - name: _airbyte_raw_dedup_cdc_excluded - - name: _airbyte_raw_dedup_exchange_rate - - name: _airbyte_raw_exchange_rate - - name: _airbyte_raw_renamed_dedup_cdc_excluded +- name: test_normalization_iakoz + quoting: + database: true + schema: false + identifier: false + tables: + - name: _airbyte_raw_dedup_cdc_excluded + - name: _airbyte_raw_dedup_exchange_rate + - name: _airbyte_raw_exchange_rate + - name: _airbyte_raw_renamed_dedup_cdc_excluded diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index fd784e2bb931..5b247c0f91b4 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,13 +1,13 @@ - delete from "integrationtests".test_normalization."dedup_exchange_rate_scd" + delete from "integrationtests".test_normalization_iakoz."dedup_exchange_rate_scd" where (_airbyte_unique_key_scd) in ( select (_airbyte_unique_key_scd) from "dedup_exchange_rate_scd__dbt_tmp" ); - insert into "integrationtests".test_normalization."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + insert into "integrationtests".test_normalization_iakoz."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") ( select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" from "dedup_exchange_rate_scd__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index 0cda4f2641d5..0f7e7f539769 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,13 +1,13 @@ - delete from "integrationtests".test_normalization."dedup_exchange_rate" + delete from "integrationtests".test_normalization_iakoz."dedup_exchange_rate" where (_airbyte_unique_key) in ( select (_airbyte_unique_key) from "dedup_exchange_rate__dbt_tmp" ); - insert into "integrationtests".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + insert into "integrationtests".test_normalization_iakoz."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") ( select "_airbyte_unique_key", "id", "currency", "date", "timestamp_col", "hkd@spéçiäl & characters", "hkd_special___characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" from "dedup_exchange_rate__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql index f2537f70055b..8f109df3f8a6 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,7 +1,7 @@ create table - "integrationtests".test_normalization."exchange_rate__dbt_tmp" + "integrationtests".test_normalization_iakoz."exchange_rate__dbt_tmp" compound sortkey(_airbyte_emitted_at) @@ -11,7 +11,7 @@ with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "integrationtests".test_normalization._airbyte_raw_exchange_rate +-- depends_on: "integrationtests".test_normalization_iakoz._airbyte_raw_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -25,7 +25,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_exchange_rate as table_alias +from "integrationtests".test_normalization_iakoz._airbyte_raw_exchange_rate as table_alias -- exchange_rate where 1 = 1 ), __dbt__cte__exchange_rate_ab2 as ( @@ -87,6 +87,6 @@ select getdate() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "integrationtests".test_normalization._airbyte_raw_exchange_rate +-- exchange_rate from "integrationtests".test_normalization_iakoz._airbyte_raw_exchange_rate where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index f20d8dbdc37b..dafa7b5500a9 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/second_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,11 +1,11 @@ - create view "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_stg__dbt_tmp" as ( + create view "integrationtests"._airbyte_test_normalization_iakoz."dedup_exchange_rate_stg__dbt_tmp" as ( with __dbt__cte__dedup_exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate +-- depends_on: "integrationtests".test_normalization_iakoz._airbyte_raw_dedup_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -18,7 +18,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +from "integrationtests".test_normalization_iakoz._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql index 0e0d5f46f449..5f37d34fd2e3 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/scd/test_normalization/dedup_exchange_rate_scd.sql @@ -1,13 +1,13 @@ - delete from "integrationtests".test_normalization."dedup_exchange_rate_scd" + delete from "integrationtests".test_normalization_iakoz."dedup_exchange_rate_scd" where (_airbyte_unique_key_scd) in ( select (_airbyte_unique_key_scd) from "dedup_exchange_rate_scd__dbt_tmp" ); - insert into "integrationtests".test_normalization."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + insert into "integrationtests".test_normalization_iakoz."dedup_exchange_rate_scd" ("_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") ( select "_airbyte_unique_key", "_airbyte_unique_key_scd", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_start_at", "_airbyte_end_at", "_airbyte_active_row", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" from "dedup_exchange_rate_scd__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql index d506183c78f7..72c0508ce9a1 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_incremental/test_normalization/dedup_exchange_rate.sql @@ -1,13 +1,13 @@ - delete from "integrationtests".test_normalization."dedup_exchange_rate" + delete from "integrationtests".test_normalization_iakoz."dedup_exchange_rate" where (_airbyte_unique_key) in ( select (_airbyte_unique_key) from "dedup_exchange_rate__dbt_tmp" ); - insert into "integrationtests".test_normalization."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") + insert into "integrationtests".test_normalization_iakoz."dedup_exchange_rate" ("_airbyte_unique_key", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid") ( select "_airbyte_unique_key", "id", "currency", "new_column", "date", "timestamp_col", "hkd@spéçiäl & characters", "nzd", "usd", "_airbyte_ab_id", "_airbyte_emitted_at", "_airbyte_normalized_at", "_airbyte_dedup_exchange_rate_hashid" from "dedup_exchange_rate__dbt_tmp" diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql index 1fa0ba1e56c4..8dc03a37f1d7 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_tables/test_normalization/exchange_rate.sql @@ -1,7 +1,7 @@ create table - "integrationtests".test_normalization."exchange_rate__dbt_tmp" + "integrationtests".test_normalization_iakoz."exchange_rate__dbt_tmp" compound sortkey(_airbyte_emitted_at) @@ -11,7 +11,7 @@ with __dbt__cte__exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "integrationtests".test_normalization._airbyte_raw_exchange_rate +-- depends_on: "integrationtests".test_normalization_iakoz._airbyte_raw_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -25,7 +25,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_exchange_rate as table_alias +from "integrationtests".test_normalization_iakoz._airbyte_raw_exchange_rate as table_alias -- exchange_rate where 1 = 1 ), __dbt__cte__exchange_rate_ab2 as ( @@ -89,6 +89,6 @@ select getdate() as _airbyte_normalized_at, _airbyte_exchange_rate_hashid from __dbt__cte__exchange_rate_ab3 --- exchange_rate from "integrationtests".test_normalization._airbyte_raw_exchange_rate +-- exchange_rate from "integrationtests".test_normalization_iakoz._airbyte_raw_exchange_rate where 1 = 1 ); \ No newline at end of file diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql index c7ee5d552f22..5e70ac9952ac 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/redshift/test_simple_streams/third_output/airbyte_views/test_normalization/dedup_exchange_rate_stg.sql @@ -1,11 +1,11 @@ - create view "integrationtests"._airbyte_test_normalization."dedup_exchange_rate_stg__dbt_tmp" as ( + create view "integrationtests"._airbyte_test_normalization_iakoz."dedup_exchange_rate_stg__dbt_tmp" as ( with __dbt__cte__dedup_exchange_rate_ab1 as ( -- SQL model to parse JSON blob stored in a single column and extract into separated field columns as described by the JSON Schema --- depends_on: "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate +-- depends_on: "integrationtests".test_normalization_iakoz._airbyte_raw_dedup_exchange_rate select case when _airbyte_data."id" != '' then _airbyte_data."id" end as id, case when _airbyte_data."currency" != '' then _airbyte_data."currency" end as currency, @@ -18,7 +18,7 @@ select _airbyte_ab_id, _airbyte_emitted_at, getdate() as _airbyte_normalized_at -from "integrationtests".test_normalization._airbyte_raw_dedup_exchange_rate as table_alias +from "integrationtests".test_normalization_iakoz._airbyte_raw_dedup_exchange_rate as table_alias -- dedup_exchange_rate where 1 = 1 diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml index 571d5bae581b..8a64d6b8085f 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize model-paths: - - models +- models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests +- tests seed-paths: - - data +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs packages-install-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: true schema: false @@ -43,10 +43,10 @@ models: +tags: airbyte_internal_views +materialized: view dispatch: - - macro_namespace: dbt_utils - search_order: - - airbyte_utils - - dbt_utils +- macro_namespace: dbt_utils + search_order: + - airbyte_utils + - dbt_utils vars: json_column: _airbyte_data models_to_source: @@ -84,6 +84,10 @@ vars: UNNEST_ALIAS_AB2: TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS UNNEST_ALIAS_AB3: TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS UNNEST_ALIAS: TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS + ARRAYS_AB1: TEST_NORMALIZATION._AIRBYTE_RAW_ARRAYS + ARRAYS_AB2: TEST_NORMALIZATION._AIRBYTE_RAW_ARRAYS + ARRAYS_AB3: TEST_NORMALIZATION._AIRBYTE_RAW_ARRAYS + ARRAYS: TEST_NORMALIZATION._AIRBYTE_RAW_ARRAYS NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB1: TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB2: TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_AB3: TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES @@ -96,6 +100,10 @@ vars: UNNEST_ALIAS_CHILDREN_AB2: TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS UNNEST_ALIAS_CHILDREN_AB3: TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS UNNEST_ALIAS_CHILDREN: TEST_NORMALIZATION._AIRBYTE_RAW_UNNEST_ALIAS + ARRAYS_NESTED_ARRAY_PARENT_AB1: TEST_NORMALIZATION._AIRBYTE_RAW_ARRAYS + ARRAYS_NESTED_ARRAY_PARENT_AB2: TEST_NORMALIZATION._AIRBYTE_RAW_ARRAYS + ARRAYS_NESTED_ARRAY_PARENT_AB3: TEST_NORMALIZATION._AIRBYTE_RAW_ARRAYS + ARRAYS_NESTED_ARRAY_PARENT: TEST_NORMALIZATION._AIRBYTE_RAW_ARRAYS NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB1: TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB2: TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES_PARTITION_DOUBLE_ARRAY_DATA_AB3: TEST_NORMALIZATION._AIRBYTE_RAW_NESTED_STREAM_WITH_COMPLEX_COLUMNS_RESULTING_INTO_LONG_NAMES diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/sources.yml index f03c2b8ae028..b51dbe4cce7a 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_nested_streams/models/generated/sources.yml @@ -6,6 +6,7 @@ sources: schema: false identifier: false tables: + - name: _AIRBYTE_RAW_ARRAYS - name: _AIRBYTE_RAW_CONFLICT_STREAM_ARRAY - name: _AIRBYTE_RAW_CONFLICT_STREAM_NAME - name: _AIRBYTE_RAW_CONFLICT_STREAM_SCALAR diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml index 76a02e4351f0..2b466206f083 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/dbt_project.yml @@ -1,25 +1,25 @@ name: airbyte_utils -version: "1.0" +version: '1.0' config-version: 2 profile: normalize model-paths: - - models +- models docs-paths: - - docs +- docs analysis-paths: - - analysis +- analysis test-paths: - - tests +- tests seed-paths: - - data +- data macro-paths: - - macros +- macros target-path: ../build log-path: ../logs packages-install-path: /dbt clean-targets: - - build - - dbt_modules +- build +- dbt_modules quoting: database: true schema: false @@ -43,10 +43,10 @@ models: +tags: airbyte_internal_views +materialized: view dispatch: - - macro_namespace: dbt_utils - search_order: - - airbyte_utils - - dbt_utils +- macro_namespace: dbt_utils + search_order: + - airbyte_utils + - dbt_utils vars: json_column: _airbyte_data models_to_source: @@ -84,3 +84,8 @@ vars: MULTIPLE_COLUMN_NAMES_CONFLICTS_STG: TEST_NORMALIZATION._AIRBYTE_RAW_MULTIPLE_COLUMN_NAMES_CONFLICTS MULTIPLE_COLUMN_NAMES_CONFLICTS_SCD: TEST_NORMALIZATION._AIRBYTE_RAW_MULTIPLE_COLUMN_NAMES_CONFLICTS MULTIPLE_COLUMN_NAMES_CONFLICTS: TEST_NORMALIZATION._AIRBYTE_RAW_MULTIPLE_COLUMN_NAMES_CONFLICTS + TYPES_TESTING_AB1: TEST_NORMALIZATION._AIRBYTE_RAW_TYPES_TESTING + TYPES_TESTING_AB2: TEST_NORMALIZATION._AIRBYTE_RAW_TYPES_TESTING + TYPES_TESTING_STG: TEST_NORMALIZATION._AIRBYTE_RAW_TYPES_TESTING + TYPES_TESTING_SCD: TEST_NORMALIZATION._AIRBYTE_RAW_TYPES_TESTING + TYPES_TESTING: TEST_NORMALIZATION._AIRBYTE_RAW_TYPES_TESTING diff --git a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml index 1d9d7ae323bb..2932fe914c6c 100644 --- a/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml +++ b/airbyte-integrations/bases/base-normalization/integration_tests/normalization_test_output/snowflake/test_simple_streams/models/generated/sources.yml @@ -13,3 +13,4 @@ sources: - name: _AIRBYTE_RAW_MULTIPLE_COLUMN_NAMES_CONFLICTS - name: _AIRBYTE_RAW_POS_DEDUP_CDCX - name: _AIRBYTE_RAW_RENAMED_DEDUP_CDC_EXCLUDED + - name: _AIRBYTE_RAW_TYPES_TESTING