Skip to content

Commit

Permalink
Update logic a little bit
Browse files Browse the repository at this point in the history
  • Loading branch information
@emielver
emielver committed Aug 16, 2023
1 parent ac5b307 commit 2e1cdb3
Show file tree
Hide file tree
Showing 3 changed files with 26 additions and 26 deletions.
8 changes: 4 additions & 4 deletions integration_tests/dbt_project.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,11 +36,11 @@ vars:
snowplow__disable_errors: true
snowplow__custom_test: false
snowplow__start_date: "2021-02-20"
snowplow__session_identifiers: [{"table" : "events", "field" : "domain_sessionid"}]
snowplow__custom_session_identifiers: [{"table": "contexts_com_snowplowanalytics_session_identifier_2_0_0", "schema": "contexts_com_snowplowanalytics_session_identifier_2_0_0", "field": "session_identifier", "prefix": "si_t", "alias": "sito"}, {"table": "contexts_com_snowplowanalytics_session_identifier_1_0_0", "schema": "contexts_com_snowplowanalytics_session_identifier_1_0_0", "field" : "session_id", "prefix" : "si_o", "alias": "sido"}]
snowplow__session_identifiers: [{"schema": "atomic", "field" : "domain_sessionid"}]
snowplow__custom_session_identifiers: [{"schema": "contexts_com_snowplowanalytics_session_identifier_2_0_0", "field": "session_identifier", "prefix": "si_t", "alias": "sito"}, {"schema": "contexts_com_snowplowanalytics_session_identifier_1_0_0", "field" : "session_id", "prefix" : "si_o", "alias": "sido"}]
snowplow__session_timestamp: 'collector_tstamp'
snowplow__user_identifiers: [{"table": "events", "field" : "domain_userid"}]
snowplow__custom_user_identifiers: [{"table": "contexts_com_snowplowanalytics_user_identifier_2_0_0", "schema": "contexts_com_snowplowanalytics_user_identifier_2_0_0", "field" : "user_id", "prefix" : "ui_t", "alias": "uidt"}, {"table": "contexts_com_snowplowanalytics_user_identifier_1_0_0", "schema": "contexts_com_snowplowanalytics_user_identifier_1_0_0", "field" : "user_id", "prefix": "ui_o", "alias": "uido"}]
snowplow__user_identifiers: [{"schema": "atomic", "field" : "domain_userid"}]
snowplow__custom_user_identifiers: [{"schema": "contexts_com_snowplowanalytics_user_identifier_2_0_0", "field" : "user_id", "prefix" : "ui_t", "alias": "uidt"}, {"schema": "contexts_com_snowplowanalytics_user_identifier_1_0_0", "field" : "user_id", "prefix": "ui_o", "alias": "uido"}]
snowplow__quarantined_sessions: 'snowplow_base_quarantined_sessions_actual'
snowplow__event_limits: 'snowplow_base_new_event_limits_actual'
snowplow__incremental_manifest: 'snowplow_incremental_manifest_actual'
Expand Down
14 changes: 7 additions & 7 deletions macros/base/base_create_snowplow_events_this_run.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
{% else -%}
COALESCE(
{% for identifier in session_identifiers %}
{%- if identifier['table']|lower != 'events' -%}
{%- if identifier['schema']|lower != 'atomic' -%}
{{ snowplow_utils.get_field(identifier['schema'], identifier['field'], 'e', dbt.type_string(), 0) }}
{%- else -%}
e.{{identifier['field']}}
Expand Down Expand Up @@ -88,8 +88,8 @@

{% if session_identifiers -%}
{% for identifier in session_identifiers %}
{% if identifier['table']|lower != 'events' %}
{{ snowplow_utils.get_sde_or_context(snowplow_events_schema, identifier['table'], lower_limit, upper_limit, identifier['prefix']) }},
{% if identifier['schema']|lower != 'atomic' %}
{{ snowplow_utils.get_sde_or_context(snowplow_events_schema, identifier['schema'], lower_limit, upper_limit, identifier['prefix']) }},
{%- endif -%}
{% endfor %}
{% endif %}
Expand Down Expand Up @@ -123,8 +123,8 @@
{% else -%}
COALESCE(
{% for identifier in session_identifiers %}
{%- if identifier['table']|lower != 'events' %}
{% if identifier['alias'] %}{{identifier['alias']}}{% else %}{{identifier['table']}}{% endif %}.{% if identifier['prefix'] %}{{ identifier['prefix'] }}{% else %}{{ identifier['table']}}{% endif %}_{{identifier['field']}}
{%- if identifier['schema']|lower != 'atomic' %}
{% if identifier['alias'] %}{{identifier['alias']}}{% else %}{{identifier['schema']}}{% endif %}.{% if identifier['prefix'] %}{{ identifier['prefix'] }}{% else %}{{ identifier['schema']}}{% endif %}_{{identifier['field']}}
{%- else %}
e.{{identifier['field']}}
{%- endif -%}
Expand All @@ -141,8 +141,8 @@
from {{ snowplow_events }} e
{% if session_identifiers|length > 0 %}
{% for identifier in session_identifiers %}
{%- if identifier['table']|lower != 'events' -%}
left join {{ identifier['table'] }} {% if identifier['alias'] %}as {{ identifier['alias'] }}{% endif %} on e.event_id = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['table'] }}{% endif %}.{{identifier['prefix']}}__id and e.collector_tstamp = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['table'] }}{% endif %}.{{ identifier['prefix'] }}__tstamp
{%- if identifier['schema']|lower != 'atomic' -%}
left join {{ identifier['schema'] }} {% if identifier['alias'] %}as {{ identifier['alias'] }}{% endif %} on e.event_id = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['schema'] }}{% endif %}.{{identifier['prefix']}}__id and e.collector_tstamp = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['schema'] }}{% endif %}.{{ identifier['prefix'] }}__tstamp
{% endif -%}
{% endfor %}
{% endif %}
Expand Down
30 changes: 15 additions & 15 deletions macros/base/base_create_snowplow_sessions_lifecycle_manifest.sql
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{# Add quarantined_sessions reference properly #}
{% macro base_create_snowplow_sessions_lifecycle_manifest(session_identifiers=[{"table" : "events", "field" : "domain_sessionid"}], session_sql=none, session_timestamp='load_tstamp', user_identifiers=[{"table" : "events", "field" : "domain_userid"}], user_sql=none, quarantined_sessions=none, derived_tstamp_partitioned=true, days_late_allowed=3, max_session_days=3, app_ids=[], snowplow_events_database=none, snowplow_events_schema='atomic', snowplow_events_table='events', event_limits_table='snowplow_base_new_event_limits', incremental_manifest_table='snowplow_incremental_manifest') %}
{% macro base_create_snowplow_sessions_lifecycle_manifest(session_identifiers=[{"schema": "atomic", "field" : "domain_sessionid"}], session_sql=none, session_timestamp='load_tstamp', user_identifiers=[{"schema": "atomic", "field" : "domain_userid"}], user_sql=none, quarantined_sessions=none, derived_tstamp_partitioned=true, days_late_allowed=3, max_session_days=3, app_ids=[], snowplow_events_database=none, snowplow_events_schema='atomic', snowplow_events_table='events', event_limits_table='snowplow_base_new_event_limits', incremental_manifest_table='snowplow_incremental_manifest') %}
{{ return(adapter.dispatch('base_create_snowplow_sessions_lifecycle_manifest', 'snowplow_utils')(session_identifiers, session_sql, session_timestamp, user_identifiers, user_sql, quarantined_sessions, derived_tstamp_partitioned, days_late_allowed, max_session_days, app_ids, snowplow_events_database, snowplow_events_schema, snowplow_events_table, event_limits_table, incremental_manifest_table)) }}
{% endmacro %}

Expand All @@ -19,7 +19,7 @@
{%- elif session_identifiers|length > 0 %}
COALESCE(
{% for identifier in session_identifiers %}
{%- if identifier['table']|lower != 'events' -%}
{%- if identifier['schema']|lower != 'atomic' -%}
{{ snowplow_utils.get_field(identifier['schema'], identifier['field'], 'e', dbt.type_string(), 0) }}
{%- else -%}
e.{{identifier['field']}}
Expand All @@ -37,7 +37,7 @@
max(
COALESCE(
{% for identifier in user_identifiers %}
{%- if identifier['table']|lower != 'events' -%}
{%- if identifier['schema']|lower != 'atomic' -%}
{{ snowplow_utils.get_field(identifier['schema'], identifier['field'], 'e', dbt.type_string(), 0) }}
{%- else -%}
e.{{identifier['field']}}
Expand Down Expand Up @@ -141,16 +141,16 @@
with
{% if session_identifiers %}
{% for identifier in session_identifiers %}
{% if identifier['table']|lower != 'events' %}
{{ snowplow_utils.get_sde_or_context(snowplow_events_schema, identifier['table'], lower_limit, upper_limit, identifier['prefix']) }},
{% if identifier['schema']|lower != 'atomic' %}
{{ snowplow_utils.get_sde_or_context(snowplow_events_schema, identifier['schema'], lower_limit, upper_limit, identifier['prefix']) }},
{%- endif -%}
{% endfor %}
{% endif %}

{% if user_identifiers%}
{% for identifier in user_identifiers %}
{% if identifier['table']|lower != 'events' %}
{{ snowplow_utils.get_sde_or_context(snowplow_events_schema, identifier['table'], lower_limit, upper_limit, identifier['prefix']) }},
{% if identifier['schema']|lower != 'atomic' %}
{{ snowplow_utils.get_sde_or_context(snowplow_events_schema, identifier['schema'], lower_limit, upper_limit, identifier['prefix']) }},
{%- endif -%}
{% endfor %}
{% endif %}
Expand All @@ -161,8 +161,8 @@
{% elif session_identifiers|length > 0 %}
COALESCE(
{% for identifier in session_identifiers %}
{%- if identifier['table']|lower != 'events' -%}
{% if identifier['alias'] %}{{identifier['alias']}}{% else %}{{identifier['table']}}{% endif %}.{% if identifier['prefix'] %}{{ identifier['prefix'] }}{% else %}{{ identifier['table']}}{% endif %}_{{identifier['field']}}
{%- if identifier['schema']|lower != 'atomic' -%}
{% if identifier['alias'] %}{{identifier['alias']}}{% else %}{{identifier['schema']}}{% endif %}.{% if identifier['prefix'] %}{{ identifier['prefix'] }}{% else %}{{ identifier['schema']}}{% endif %}_{{identifier['field']}}
{%- else -%}
e.{{identifier['field']}}
{%- endif -%}
Expand All @@ -177,8 +177,8 @@
max(
COALESCE(
{% for identifier in user_identifiers %}
{%- if identifier['table']|lower != 'events' %}
{% if identifier['alias'] %}{{identifier['alias']}}{% else %}{{identifier['table']}}{% endif %}.{% if identifier['prefix'] %}{{ identifier['prefix'] }}{% else %}{{ identifier['table']}}{% endif %}_{{identifier['field']}}
{%- if identifier['schema']|lower != 'atomic' %}
{% if identifier['alias'] %}{{identifier['alias']}}{% else %}{{identifier['schema']}}{% endif %}.{% if identifier['prefix'] %}{{ identifier['prefix'] }}{% else %}{{ identifier['schema']}}{% endif %}_{{identifier['field']}}
{%- else %}
e.{{identifier['field']}}
{%- endif -%}
Expand All @@ -198,15 +198,15 @@
from {{ snowplow_events }} e
{% if session_identifiers|length > 0 %}
{% for identifier in session_identifiers %}
{%- if identifier['table']|lower != 'events' -%}
left join {{ identifier['table'] }} {% if identifier['alias'] %}as {{ identifier['alias'] }}{% endif %} on e.event_id = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['table'] }}{% endif %}.{{identifier['prefix']}}__id and e.collector_tstamp = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['table'] }}{% endif %}.{{ identifier['prefix'] }}__tstamp
{%- if identifier['schema']|lower != 'atomic' -%}
left join {{ identifier['schema'] }} {% if identifier['alias'] %}as {{ identifier['alias'] }}{% endif %} on e.event_id = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['schema'] }}{% endif %}.{{identifier['prefix']}}__id and e.collector_tstamp = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['schema'] }}{% endif %}.{{ identifier['prefix'] }}__tstamp
{% endif -%}
{% endfor %}
{% endif %}
{% if session_identifiers|length > 0 %}
{% for identifier in user_identifiers %}
{%- if identifier['table']|lower != 'events' -%}
left join {{ identifier['table'] }} {% if identifier['alias'] %}as {{ identifier['alias'] }}{% endif %} on e.event_id = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['table'] }}{% endif %}.{{identifier['prefix']}}__id and e.collector_tstamp = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['table'] }}{% endif %}.{{ identifier['prefix'] }}__tstamp
{%- if identifier['schema']|lower != 'atomic' -%}
left join {{ identifier['schema'] }} {% if identifier['alias'] %}as {{ identifier['alias'] }}{% endif %} on e.event_id = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['schema'] }}{% endif %}.{{identifier['prefix']}}__id and e.collector_tstamp = {% if identifier['alias'] %}{{ identifier['alias']}}{% else %}{{ identifier['schema'] }}{% endif %}.{{ identifier['prefix'] }}__tstamp
{% endif -%}
{% endfor %}
{% endif %}
Expand Down

0 comments on commit 2e1cdb3

Please sign in to comment.