diff --git a/macros/generic/mandatory.sql b/macros/generic/mandatory.sql index 9a3611d..09b8ea9 100644 --- a/macros/generic/mandatory.sql +++ b/macros/generic/mandatory.sql @@ -1,21 +1,21 @@ -{%- macro mandatory(source_table, mandatory_column, data_type) -%} +{%- macro mandatory(relation, mandatory_column, data_type) -%} {%- if data_type == 'boolean' -%} - {{ pm_utils.to_boolean(mandatory_column, source_table) }} + {{ pm_utils.to_boolean(mandatory_column, relation) }} {%- elif data_type == 'date' -%} - {{ pm_utils.to_date(mandatory_column, source_table) }} + {{ pm_utils.to_date(mandatory_column, relation) }} {%- elif data_type == 'double' -%} - {{ pm_utils.to_double(mandatory_column, source_table) }} + {{ pm_utils.to_double(mandatory_column, relation) }} {%- elif data_type == 'integer' -%} - {{ pm_utils.to_integer(mandatory_column, source_table) }} + {{ pm_utils.to_integer(mandatory_column, relation) }} {%- elif data_type == 'time' -%} - {{ pm_utils.to_time(mandatory_column, source_table) }} + {{ pm_utils.to_time(mandatory_column, relation) }} {%- elif data_type == 'datetime' -%} - {{ pm_utils.to_timestamp(mandatory_column, source_table) }} + {{ pm_utils.to_timestamp(mandatory_column, relation) }} {%- elif data_type == 'text' -%} {{ pm_utils.to_varchar(mandatory_column) }} {%- elif data_type == 'id' -%} - {{ pm_utils.to_integer(mandatory_column, source_table) }} + {{ pm_utils.to_integer(mandatory_column, relation) }} {%- else -%} {{ pm_utils.to_varchar(mandatory_column) }} {%- endif -%} diff --git a/macros/generic/optional.sql b/macros/generic/optional.sql index a8cd7c9..46e01c5 100644 --- a/macros/generic/optional.sql +++ b/macros/generic/optional.sql @@ -1,6 +1,6 @@ -{%- macro optional(source_table, optional_column, data_type) -%} +{%- macro optional(relation, optional_column, data_type) -%} -{%- set columns = adapter.get_columns_in_relation(source_table) -%} +{%- set columns = adapter.get_columns_in_relation(relation) -%} {# Create list of column names.#} {%- set column_names = [] -%} @@ -16,22 +16,22 @@ {%- endif -%} {%- if data_type == 'boolean' -%} - {{ pm_utils.to_boolean(column_value, source_table) }} + {{ pm_utils.to_boolean(column_value, relation) }} {%- elif data_type == 'date' -%} - {{ pm_utils.to_date(column_value, source_table) }} + {{ pm_utils.to_date(column_value, relation) }} {%- elif data_type == 'double' -%} - {{ pm_utils.to_double(column_value, source_table) }} + {{ pm_utils.to_double(column_value, relation) }} {%- elif data_type == 'integer' -%} - {{ pm_utils.to_integer(column_value, source_table) }} + {{ pm_utils.to_integer(column_value, relation) }} {%- elif data_type == 'time' -%} - {{ pm_utils.to_time(column_value, source_table) }} + {{ pm_utils.to_time(column_value, relation) }} {%- elif data_type == 'datetime' -%} - {{ pm_utils.to_timestamp(column_value, source_table) }} + {{ pm_utils.to_timestamp(column_value, relation) }} {%- elif data_type == 'text' -%} {{ pm_utils.to_varchar(column_value) }} {%- elif data_type == 'id' -%} {% if optional_column in column_names %} - {{ pm_utils.to_integer(column_value, source_table) }} + {{ pm_utils.to_integer(column_value, relation) }} {% else %} row_number() over (order by (select null)) {% endif%} diff --git a/macros/multiple_databases/to_boolean.sql b/macros/multiple_databases/to_boolean.sql index 40b2a1c..ac9ab3b 100644 --- a/macros/multiple_databases/to_boolean.sql +++ b/macros/multiple_databases/to_boolean.sql @@ -1,4 +1,4 @@ -{%- macro to_boolean(field, table) -%} +{%- macro to_boolean(field, relation) -%} {%- if target.type == 'snowflake' -%} {%- if field in ('true', 'false', '1', '0') -%} @@ -15,11 +15,11 @@ {%- endif -%} {# Warning if type casting will introduce null values for at least 1 record. #} -{% if table is defined %} +{% if relation is defined %} {% set query %} select count(*) as "record_count" - from "{{ table.database }}"."{{ table.schema }}"."{{ table.identifier }}" + from "{{ relation.database }}"."{{ relation.schema }}"."{{ relation.identifier }}" where {{ field }} is not null and {% if target.type == 'snowflake' -%} {%- if field in ('true', 'false', '1', '0') -%} @@ -45,7 +45,7 @@ {% if record_count > 0 %} {% if var("log_result", False) == True %} - {{ log(tojson({'Key': 'ConvertBoolean', 'Details': {'table_identifier': table.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ table.identifier ~ '.' ~ field ~ '\' to a boolean for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} + {{ log(tojson({'Key': 'ConvertBoolean', 'Details': {'relation_identifier': relation.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ relation.identifier ~ '.' ~ field ~ '\' to a boolean for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} {% endif %} {% endif %} {% endif %} diff --git a/macros/multiple_databases/to_date.sql b/macros/multiple_databases/to_date.sql index 1ed1cec..d0e0cf9 100644 --- a/macros/multiple_databases/to_date.sql +++ b/macros/multiple_databases/to_date.sql @@ -1,4 +1,4 @@ -{%- macro to_date(field, table) -%} +{%- macro to_date(field, relation) -%} {# Cast to date when the input is in a date or a datetime format. This is default behavior for SQL Server. #} {%- if target.type == 'snowflake' -%} @@ -12,11 +12,11 @@ {%- endif -%} {# Warning if type casting will introduce null values for at least 1 record. #} -{% if table is defined %} +{% if relation is defined %} {% set query %} select count(*) as "record_count" - from "{{ table.database }}"."{{ table.schema }}"."{{ table.identifier }}" + from "{{ relation.database }}"."{{ relation.schema }}"."{{ relation.identifier }}" where {{ field }} is not null and {% if target.type == 'snowflake' -%} case @@ -38,7 +38,7 @@ {% if record_count > 0 %} {% if var("log_result", False) == True %} - {{ log(tojson({'Key': 'ConvertDate', 'Details': {'table_identifier': table.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ table.identifier ~ '.' ~ field ~ '\' to a date for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} + {{ log(tojson({'Key': 'ConvertDate', 'Details': {'relation_identifier': relation.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ relation.identifier ~ '.' ~ field ~ '\' to a date for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} {% endif %} {% endif %} {% endif %} diff --git a/macros/multiple_databases/to_double.sql b/macros/multiple_databases/to_double.sql index 9ea0692..121e4fa 100644 --- a/macros/multiple_databases/to_double.sql +++ b/macros/multiple_databases/to_double.sql @@ -1,4 +1,4 @@ -{%- macro to_double(field, table) -%} +{%- macro to_double(field, relation) -%} {%- if target.type == 'snowflake' -%} try_to_double(to_varchar({{ field }})) @@ -7,11 +7,11 @@ {%- endif -%} {# Warning if type casting will introduce null values for at least 1 record. #} -{% if table is defined %} +{% if relation is defined %} {% set query %} select count(*) as "record_count" - from "{{ table.database }}"."{{ table.schema }}"."{{ table.identifier }}" + from "{{ relation.database }}"."{{ relation.schema }}"."{{ relation.identifier }}" where {{ field }} is not null and {% if target.type == 'snowflake' -%} try_to_double(to_varchar({{ field }})) is null @@ -29,7 +29,7 @@ {% if record_count > 0 %} {% if var("log_result", False) == True %} - {{ log(tojson({'Key': 'ConvertDouble', 'Details': {'table_identifier': table.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ table.identifier ~ '.' ~ field ~ '\' to a double for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} + {{ log(tojson({'Key': 'ConvertDouble', 'Details': {'relation_identifier': relation.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ relation.identifier ~ '.' ~ field ~ '\' to a double for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} {% endif %} {% endif %} {% endif %} diff --git a/macros/multiple_databases/to_integer.sql b/macros/multiple_databases/to_integer.sql index 5a19237..c11625c 100644 --- a/macros/multiple_databases/to_integer.sql +++ b/macros/multiple_databases/to_integer.sql @@ -1,4 +1,4 @@ -{%- macro to_integer(field, table) -%} +{%- macro to_integer(field, relation) -%} {%- if target.type == 'snowflake' -%} try_to_number(to_varchar({{ field }})) @@ -7,11 +7,11 @@ {%- endif -%} {# Warning if type casting will introduce null values for at least 1 record. #} -{% if table is defined %} +{% if relation is defined %} {% set query %} select count(*) as "record_count" - from "{{ table.database }}"."{{ table.schema }}"."{{ table.identifier }}" + from "{{ relation.database }}"."{{ relation.schema }}"."{{ relation.identifier }}" where {{ field }} is not null and {% if target.type == 'snowflake' -%} try_to_number(to_varchar({{ field }})) is null @@ -29,7 +29,7 @@ {% if record_count > 0 %} {% if var("log_result", False) == True %} - {{ log(tojson({'Key': 'ConvertInteger', 'Details': {'table_identifier': table.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ table.identifier ~ '.' ~ field ~ '\' to an integer for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} + {{ log(tojson({'Key': 'ConvertInteger', 'Details': {'relation_identifier': relation.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ relation.identifier ~ '.' ~ field ~ '\' to an integer for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} {% endif %} {% endif %} {% endif %} diff --git a/macros/multiple_databases/to_time.sql b/macros/multiple_databases/to_time.sql index 1b07fa2..23d2c4b 100644 --- a/macros/multiple_databases/to_time.sql +++ b/macros/multiple_databases/to_time.sql @@ -1,4 +1,4 @@ -{%- macro to_time(field, table) -%} +{%- macro to_time(field, relation) -%} {%- if target.type == 'snowflake' -%} try_to_time(to_varchar({{ field }}), '{{ var("time_format", "hh24:mi:ss.ff3") }}') @@ -7,11 +7,11 @@ {%- endif -%} {# Warning if type casting will introduce null values for at least 1 record. #} -{% if table is defined %} +{% if relation is defined %} {% set query %} select count(*) as "record_count" - from "{{ table.database }}"."{{ table.schema }}"."{{ table.identifier }}" + from "{{ relation.database }}"."{{ relation.schema }}"."{{ relation.identifier }}" where {{ field }} is not null and {% if target.type == 'snowflake' -%} try_to_time(to_varchar({{ field }}), '{{ var("time_format", "hh24:mi:ss.ff3") }}') is null @@ -29,7 +29,7 @@ {% if record_count > 0 %} {% if var("log_result", False) == True %} - {{ log(tojson({'Key': 'ConvertTime', 'Details': {'table_identifier': table.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ table.identifier ~ '.' ~ field ~ '\' to a time for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} + {{ log(tojson({'Key': 'ConvertTime', 'Details': {'relation_identifier': relation.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ relation.identifier ~ '.' ~ field ~ '\' to a time for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} {% endif %} {% endif %} {% endif %} diff --git a/macros/multiple_databases/to_timestamp.sql b/macros/multiple_databases/to_timestamp.sql index 19487a4..a0738c3 100644 --- a/macros/multiple_databases/to_timestamp.sql +++ b/macros/multiple_databases/to_timestamp.sql @@ -1,4 +1,4 @@ -{%- macro to_timestamp(field, table) -%} +{%- macro to_timestamp(field, relation) -%} {%- if target.type == 'snowflake' -%} try_to_timestamp(to_varchar({{ field }}), '{{ var("datetime_format", "YYYY-MM-DD hh24:mi:ss.ff3") }}') @@ -7,11 +7,11 @@ {%- endif -%} {# Warning if type casting will introduce null values for at least 1 record. #} -{% if table is defined %} +{% if relation is defined %} {% set query %} select count(*) as "record_count" - from "{{ table.database }}"."{{ table.schema }}"."{{ table.identifier }}" + from "{{ relation.database }}"."{{ relation.schema }}"."{{ relation.identifier }}" where {{ field }} is not null and {% if target.type == 'snowflake' -%} try_to_timestamp(to_varchar({{ field }}), '{{ var("datetime_format", "YYYY-MM-DD hh24:mi:ss.ff3") }}') is null @@ -29,7 +29,7 @@ {% if record_count > 0 %} {% if var("log_result", False) == True %} - {{ log(tojson({'Key': 'ConvertDatetime', 'Details': {'table_identifier': table.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ table.identifier ~ '.' ~ field ~ '\' to a datetime for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} + {{ log(tojson({'Key': 'ConvertDatetime', 'Details': {'relation_identifier': relation.identifier, 'field': field, 'record_count': record_count|string}, 'Category': 'UserWarning', 'Message': 'Failed to convert \'' ~ relation.identifier ~ '.' ~ field ~ '\' to a datetime for ' ~ record_count ~ ' records. Their values are set to NULL.'}), True) }} {% endif %} {% endif %} {% endif %}