From 00bbd8c1a08f696453f44f4074325ca169db136c Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Thu, 21 Sep 2023 17:22:19 +0300 Subject: [PATCH 1/9] criteria + normalized status --- elementary/monitor/alerts/source_freshness.py | 35 +++++----- .../alerts/get_source_freshness_alerts.sql | 40 ++++++------ .../normalized_source_freshness_status.sql | 7 ++ elementary/monitor/fetchers/tests/schema.py | 64 +++++++++++++++++++ 4 files changed, 109 insertions(+), 37 deletions(-) create mode 100644 elementary/monitor/dbt_project/macros/utils/normalized_source_freshness_status.sql diff --git a/elementary/monitor/alerts/source_freshness.py b/elementary/monitor/alerts/source_freshness.py index 085b17592..1155b807b 100644 --- a/elementary/monitor/alerts/source_freshness.py +++ b/elementary/monitor/alerts/source_freshness.py @@ -20,9 +20,10 @@ def __init__( max_loaded_at_time_ago_in_s: Optional[float], source_name: str, identifier: str, - freshness_error_after: str, - freshness_warn_after: str, - freshness_filter: str, + normalized_status: str, + error_after: str, + warn_after: str, + filter: Optional[str], path: str, error: str, **kwargs, @@ -42,9 +43,10 @@ def __init__( self.max_loaded_at_time_ago_in_s = max_loaded_at_time_ago_in_s self.source_name = source_name self.identifier = identifier - self.freshness_error_after = freshness_error_after - self.freshness_warn_after = freshness_warn_after - self.freshness_filter = freshness_filter + self.normalized_status = normalized_status + self.error_after = error_after + self.warn_after = warn_after + self.filter = filter self.path = path self.error = error self.alerts_table = SourceFreshnessAlert.TABLE_NAME @@ -55,7 +57,7 @@ def to_slack(self, is_slack_workflow: bool = False) -> SlackMessageSchema: subscribers = self.slack_message_builder.prettify_and_dedup_list( self.subscribers or [] ) - icon = self.slack_message_builder.get_slack_status_icon(self.status) + icon = self.slack_message_builder.get_slack_status_icon(self.normalized_status) title = [ self.slack_message_builder.create_header_block( @@ -68,7 +70,7 @@ def to_slack(self, is_slack_workflow: bool = False) -> SlackMessageSchema: self.slack_message_builder.create_context_block( [ f"*Source:* {self.source_name}.{self.identifier} |", - f"*Status:* {self.status}", + f"*Status:* {self.normalized_status}", ], ), self.slack_message_builder.create_context_block( @@ -84,7 +86,7 @@ def to_slack(self, is_slack_workflow: bool = False) -> SlackMessageSchema: self.slack_message_builder.create_context_block( [ f"*Source:* {self.source_name}.{self.identifier} |", - f"*Status:* {self.status} |", + f"*Status:* {self.normalized_status} |", f"*{self.detected_at_str}*", ], ), @@ -123,32 +125,31 @@ def to_slack(self, is_slack_workflow: bool = False) -> SlackMessageSchema: ) configuration = [] - if self.freshness_error_after: + + if self.error_after: configuration.append( self.slack_message_builder.create_context_block(["*Error after*"]) ) configuration.append( self.slack_message_builder.create_text_section_block( - f"`{self.freshness_error_after}`" + f"`{self.error_after}`" ) ) - if self.freshness_warn_after: + if self.warn_after: configuration.append( self.slack_message_builder.create_context_block(["*Warn after*"]) ) configuration.append( self.slack_message_builder.create_text_section_block( - f"`{self.freshness_warn_after}`" + f"`{self.warn_after}`" ) ) - if self.freshness_filter: + if self.filter: configuration.append( self.slack_message_builder.create_context_block(["*Filter*"]) ) configuration.append( - self.slack_message_builder.create_text_section_block( - f"`{self.freshness_filter}`" - ) + self.slack_message_builder.create_text_section_block(f"`{self.filter}`") ) if self.path: configuration.append( diff --git a/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql index c525ab1a3..7485f66f0 100644 --- a/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql @@ -2,24 +2,14 @@ -- depends_on: {{ ref('alerts_source_freshness') }} {% set select_pending_alerts_query %} with alerts_in_time_limit as ( - select * from {{ ref('alerts_source_freshness') }} + select *, {{ elementary_cli.normalized_source_freshness_status()}} from {{ ref('alerts_source_freshness') }} where {{ elementary.edr_cast_as_timestamp('detected_at') }} >= {{ elementary_cli.get_alerts_time_limit(days_back) }} ), - models as ( - select * from {{ ref('elementary', 'dbt_models') }} - ), - sources as ( select * from {{ ref('elementary', 'dbt_sources') }} ), - artifacts_meta as ( - select unique_id, meta from models - union all - select unique_id, meta from sources - ), - extended_alerts as ( select alerts_in_time_limit.alert_id, @@ -28,7 +18,8 @@ alerts_in_time_limit.detected_at, alerts_in_time_limit.max_loaded_at_time_ago_in_s, alerts_in_time_limit.status, - alerts_in_time_limit.error, + alerts_in_time_limit.normalized_status, + alerts_in_time_limit.result_description, alerts_in_time_limit.unique_id, {# Currently alert_class_id equals to unique_id - might change in the future so we return both #} alerts_in_time_limit.unique_id as alert_class_id, @@ -36,9 +27,9 @@ alerts_in_time_limit.schema_name, alerts_in_time_limit.source_name, alerts_in_time_limit.identifier, - alerts_in_time_limit.freshness_error_after, - alerts_in_time_limit.freshness_warn_after, - alerts_in_time_limit.freshness_filter, + alerts_in_time_limit.warn_after, + alerts_in_time_limit.error_after, + alerts_in_time_limit.filter, alerts_in_time_limit.tags, alerts_in_time_limit.meta, alerts_in_time_limit.owner, @@ -51,9 +42,12 @@ else suppression_status end as suppression_status, alerts_in_time_limit.sent_at, - artifacts_meta.meta as model_meta + sources.meta as model_meta, + sources.freshness_error_after, + sources.freshness_warn_after, + sources.freshness_filter from alerts_in_time_limit - left join artifacts_meta on alerts_in_time_limit.unique_id = artifacts_meta.unique_id + left join sources on alerts_in_time_limit.unique_id = sources.unique_id ) select * @@ -65,6 +59,10 @@ {% set alerts_dicts = elementary.agate_to_dicts(alerts_agate) %} {% set pending_alerts = [] %} {% for alert_dict in alerts_dicts %} + {% set error_after = alert_dict.get('error_after') %} + {% set warn_after = alert_dict.get('warn_after') %} + {% set filter = alert_dict.get('filter') %} + {% set pending_alert_dict = {'id': alert_dict.get('alert_id'), 'model_unique_id': alert_dict.get('unique_id'), 'alert_class_id': alert_dict.get('alert_class_id'), @@ -76,13 +74,15 @@ 'schema_name': alert_dict.get('schema_name'), 'source_name': alert_dict.get('source_name'), 'identifier': alert_dict.get('identifier'), - 'freshness_error_after': alert_dict.get('freshness_error_after'), - 'freshness_warn_after': alert_dict.get('freshness_warn_after'), - 'freshness_filter': alert_dict.get('freshness_filter'), + 'error_after': error_after if error_after is not none else alert_dict.get('freshness_error_after'), + 'warn_after': warn_after if warn_after is not none else alert_dict.get('freshness_warn_after'), + 'filter': filter if filter is not none else alert_dict.get('freshness_filter'), 'status': alert_dict.get('status'), + 'normalized_status': alert_dict.get('normalized_status'), 'owners': alert_dict.get('owner'), 'path': alert_dict.get('path'), 'error': alert_dict.get('error'), + 'result_description': alert_dict.get('result_description'), 'tags': alert_dict.get('tags'), 'model_meta': alert_dict.get('model_meta'), 'suppression_status': alert_dict.get('suppression_status'), diff --git a/elementary/monitor/dbt_project/macros/utils/normalized_source_freshness_status.sql b/elementary/monitor/dbt_project/macros/utils/normalized_source_freshness_status.sql new file mode 100644 index 000000000..d8ebab057 --- /dev/null +++ b/elementary/monitor/dbt_project/macros/utils/normalized_source_freshness_status.sql @@ -0,0 +1,7 @@ +{% macro normalized_source_freshness_status() %} + case + when status = 'error' then 'fail' + when status = 'runtime error' then 'error' + else status + end as normalized_status +{% endmacro %} diff --git a/elementary/monitor/fetchers/tests/schema.py b/elementary/monitor/fetchers/tests/schema.py index 289e88cb7..9b9962560 100644 --- a/elementary/monitor/fetchers/tests/schema.py +++ b/elementary/monitor/fetchers/tests/schema.py @@ -80,3 +80,67 @@ def parse_failures(cls, failures, values): test_type = values.get("test_type") # Elementary's tests doesn't return correct failures. return failures or None if test_type == "dbt_test" else None + + +class SourceFreshnessResultDBRowSchema(ExtendedBaseModel): + __test__ = False # Mark for pytest - The class name starts with "Test" which throws warnings on pytest runs + + source_freshness_execution_id: str + unique_id: str + max_loaded_at: Optional[str] = None + generated_at: str + execute_started_at: Optional[str] = None + status: str + normalized_status: str + error: Optional[str] = None + invocation_id: str + database_name: Optional[str] = None + schema_name: str + source_name: str + table_name: str + test_type: str + test_sub_type: str + loaded_at_field: str + meta: dict + owners: Optional[List[str]] + tags: Optional[List[str]] + error_after: dict + warn_after: dict + filter: Optional[str] = None + relation_name: str + invocations_rank_index: int + + class Config: + smart_union = True + + @validator("generated_at", pre=True) + def format_generated_at(cls, generated_at): + return convert_partial_iso_format_to_full_iso_format(generated_at) + + @validator("max_loaded_at", pre=True) + def format_max_loaded_at(cls, max_loaded_at): + return convert_partial_iso_format_to_full_iso_format(max_loaded_at) + + @validator("execute_started_at", pre=True) + def format_execute_started_at(cls, execute_started_at): + return convert_partial_iso_format_to_full_iso_format(execute_started_at) + + @validator("meta", pre=True) + def load_meta(cls, meta): + return cls._load_var_to_dict(meta) + + @validator("tags", pre=True) + def load_tags(cls, tags): + return cls._load_var_to_list(tags) + + @validator("owners", pre=True) + def load_owners(cls, owners): + return cls._load_var_to_list(owners) + + @validator("error_after", pre=True) + def load_error_after(cls, error_after): + return cls._load_var_to_dict(error_after) + + @validator("warn_after", pre=True) + def load_warn_after(cls, warn_after): + return cls._load_var_to_dict(warn_after) From 0cf58545f145e4a25bc0c135d6c7699e2b6eb9f4 Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Tue, 26 Sep 2023 09:58:47 +0300 Subject: [PATCH 2/9] use error --- .../dbt_project/macros/alerts/get_source_freshness_alerts.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql index 7485f66f0..77fb58bc2 100644 --- a/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql @@ -19,7 +19,7 @@ alerts_in_time_limit.max_loaded_at_time_ago_in_s, alerts_in_time_limit.status, alerts_in_time_limit.normalized_status, - alerts_in_time_limit.result_description, + alerts_in_time_limit.error, alerts_in_time_limit.unique_id, {# Currently alert_class_id equals to unique_id - might change in the future so we return both #} alerts_in_time_limit.unique_id as alert_class_id, @@ -82,7 +82,6 @@ 'owners': alert_dict.get('owner'), 'path': alert_dict.get('path'), 'error': alert_dict.get('error'), - 'result_description': alert_dict.get('result_description'), 'tags': alert_dict.get('tags'), 'model_meta': alert_dict.get('model_meta'), 'suppression_status': alert_dict.get('suppression_status'), From eed4ae79ff47f281dadb594fa6add282caa981da Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Tue, 26 Sep 2023 11:31:12 +0300 Subject: [PATCH 3/9] cleanup --- elementary/monitor/fetchers/tests/schema.py | 64 --------------------- 1 file changed, 64 deletions(-) diff --git a/elementary/monitor/fetchers/tests/schema.py b/elementary/monitor/fetchers/tests/schema.py index 9b9962560..289e88cb7 100644 --- a/elementary/monitor/fetchers/tests/schema.py +++ b/elementary/monitor/fetchers/tests/schema.py @@ -80,67 +80,3 @@ def parse_failures(cls, failures, values): test_type = values.get("test_type") # Elementary's tests doesn't return correct failures. return failures or None if test_type == "dbt_test" else None - - -class SourceFreshnessResultDBRowSchema(ExtendedBaseModel): - __test__ = False # Mark for pytest - The class name starts with "Test" which throws warnings on pytest runs - - source_freshness_execution_id: str - unique_id: str - max_loaded_at: Optional[str] = None - generated_at: str - execute_started_at: Optional[str] = None - status: str - normalized_status: str - error: Optional[str] = None - invocation_id: str - database_name: Optional[str] = None - schema_name: str - source_name: str - table_name: str - test_type: str - test_sub_type: str - loaded_at_field: str - meta: dict - owners: Optional[List[str]] - tags: Optional[List[str]] - error_after: dict - warn_after: dict - filter: Optional[str] = None - relation_name: str - invocations_rank_index: int - - class Config: - smart_union = True - - @validator("generated_at", pre=True) - def format_generated_at(cls, generated_at): - return convert_partial_iso_format_to_full_iso_format(generated_at) - - @validator("max_loaded_at", pre=True) - def format_max_loaded_at(cls, max_loaded_at): - return convert_partial_iso_format_to_full_iso_format(max_loaded_at) - - @validator("execute_started_at", pre=True) - def format_execute_started_at(cls, execute_started_at): - return convert_partial_iso_format_to_full_iso_format(execute_started_at) - - @validator("meta", pre=True) - def load_meta(cls, meta): - return cls._load_var_to_dict(meta) - - @validator("tags", pre=True) - def load_tags(cls, tags): - return cls._load_var_to_list(tags) - - @validator("owners", pre=True) - def load_owners(cls, owners): - return cls._load_var_to_list(owners) - - @validator("error_after", pre=True) - def load_error_after(cls, error_after): - return cls._load_var_to_dict(error_after) - - @validator("warn_after", pre=True) - def load_warn_after(cls, warn_after): - return cls._load_var_to_dict(warn_after) From 71cd38994b6af9cd7abf533a6fbd3fa808f44b55 Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Tue, 26 Sep 2023 11:51:02 +0300 Subject: [PATCH 4/9] fix mock objects --- .../monitor/api/alerts/test_alert_filters.py | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/tests/unit/monitor/api/alerts/test_alert_filters.py b/tests/unit/monitor/api/alerts/test_alert_filters.py index 42d69849b..9366a12b4 100644 --- a/tests/unit/monitor/api/alerts/test_alert_filters.py +++ b/tests/unit/monitor/api/alerts/test_alert_filters.py @@ -171,14 +171,15 @@ def initial_alerts(): tags='["one", "two"]', owners='["jeff", "john"]', status="error", + normalized_status="fail", snapshotted_at="2023-08-15T12:26:06.884065+00:00", max_loaded_at="1969-12-31T00:00:00+00:00", max_loaded_at_time_ago_in_s=1692188766.884065, source_name="elementary_integration_tests", identifier="any_type_column_anomalies_validation", - freshness_error_after='{"count": null, "period": null}', - freshness_warn_after='{"count": 1, "period": "minute"}', - freshness_filter="null", + error_after='{"count": null, "period": null}', + warn_after='{"count": 1, "period": "minute"}', + filter="null", error="problemz", ), SourceFreshnessAlert( @@ -196,14 +197,15 @@ def initial_alerts(): tags='["one", "two"]', owners='["jeff", "john"]', status="warn", + normalized_status="warn", snapshotted_at="2023-08-15T12:26:06.884065+00:00", max_loaded_at="1969-12-31T00:00:00+00:00", max_loaded_at_time_ago_in_s=1692188766.884065, source_name="elementary_integration_tests", identifier="any_type_column_anomalies_validation", - freshness_error_after='{"count": null, "period": null}', - freshness_warn_after='{"count": 1, "period": "minute"}', - freshness_filter="null", + error_after='{"count": null, "period": null}', + warn_after='{"count": 1, "period": "minute"}', + filter="null", error="problemz", ), SourceFreshnessAlert( @@ -221,14 +223,15 @@ def initial_alerts(): tags='["one", "two"]', owners='["jeff", "john"]', status="runtime error", + normalized_status="error", snapshotted_at="2023-08-15T12:26:06.884065+00:00", max_loaded_at="1969-12-31T00:00:00+00:00", max_loaded_at_time_ago_in_s=1692188766.884065, source_name="elementary_integration_tests", identifier="any_type_column_anomalies_validation", - freshness_error_after='{"count": null, "period": null}', - freshness_warn_after='{"count": 1, "period": "minute"}', - freshness_filter="null", + error_after='{"count": null, "period": null}', + warn_after='{"count": 1, "period": "minute"}', + filter="null", error="problemz", ), ] From b47260178f15b8d8fee63f42a92c98b756a8a706 Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Wed, 27 Sep 2023 17:45:56 +0300 Subject: [PATCH 5/9] query new field --- elementary/monitor/alerts/source_freshness.py | 1 + .../macros/alerts/get_source_freshness_alerts.sql | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/elementary/monitor/alerts/source_freshness.py b/elementary/monitor/alerts/source_freshness.py index 1155b807b..8cce6dc64 100644 --- a/elementary/monitor/alerts/source_freshness.py +++ b/elementary/monitor/alerts/source_freshness.py @@ -26,6 +26,7 @@ def __init__( filter: Optional[str], path: str, error: str, + freshness_description: Optional[str] = None, **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql index 77fb58bc2..4804b06e0 100644 --- a/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql @@ -45,7 +45,8 @@ sources.meta as model_meta, sources.freshness_error_after, sources.freshness_warn_after, - sources.freshness_filter + sources.freshness_filter, + sources.freshness_description from alerts_in_time_limit left join sources on alerts_in_time_limit.unique_id = sources.unique_id ) @@ -85,7 +86,8 @@ 'tags': alert_dict.get('tags'), 'model_meta': alert_dict.get('model_meta'), 'suppression_status': alert_dict.get('suppression_status'), - 'sent_at': alert_dict.get('sent_at') + 'sent_at': alert_dict.get('sent_at'), + 'freshness_description': alert_dict.get('freshness_description') } %} {% do pending_alerts.append(pending_alert_dict) %} {% endfor %} From 5af234ffad10e4db290aa19b895ed46c255948cf Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Wed, 27 Sep 2023 17:46:23 +0300 Subject: [PATCH 6/9] add description to alert --- elementary/monitor/alerts/source_freshness.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/elementary/monitor/alerts/source_freshness.py b/elementary/monitor/alerts/source_freshness.py index 8cce6dc64..3fdeb099a 100644 --- a/elementary/monitor/alerts/source_freshness.py +++ b/elementary/monitor/alerts/source_freshness.py @@ -50,6 +50,7 @@ def __init__( self.filter = filter self.path = path self.error = error + self.freshness_description = freshness_description self.alerts_table = SourceFreshnessAlert.TABLE_NAME def to_slack(self, is_slack_workflow: bool = False) -> SlackMessageSchema: @@ -101,6 +102,24 @@ def to_slack(self, is_slack_workflow: bool = False) -> SlackMessageSchema: ] ) + if self.freshness_description: + preview.extend( + [ + self.slack_message_builder.create_text_section_block( + "*Description*" + ), + self.slack_message_builder.create_context_block( + [self.freshness_description] + ), + ] + ) + else: + preview.append( + self.slack_message_builder.create_text_section_block( + "*Description*\n_No description_" + ) + ) + result = [] if self.status == "runtime error": result.extend( From bc0b99c2b8ae5ab8916a4435b0fde7c02efeb865 Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Wed, 27 Sep 2023 17:47:13 +0300 Subject: [PATCH 7/9] Revert "add description to alert" This reverts commit 5af234ffad10e4db290aa19b895ed46c255948cf. --- elementary/monitor/alerts/source_freshness.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/elementary/monitor/alerts/source_freshness.py b/elementary/monitor/alerts/source_freshness.py index 3fdeb099a..8cce6dc64 100644 --- a/elementary/monitor/alerts/source_freshness.py +++ b/elementary/monitor/alerts/source_freshness.py @@ -50,7 +50,6 @@ def __init__( self.filter = filter self.path = path self.error = error - self.freshness_description = freshness_description self.alerts_table = SourceFreshnessAlert.TABLE_NAME def to_slack(self, is_slack_workflow: bool = False) -> SlackMessageSchema: @@ -102,24 +101,6 @@ def to_slack(self, is_slack_workflow: bool = False) -> SlackMessageSchema: ] ) - if self.freshness_description: - preview.extend( - [ - self.slack_message_builder.create_text_section_block( - "*Description*" - ), - self.slack_message_builder.create_context_block( - [self.freshness_description] - ), - ] - ) - else: - preview.append( - self.slack_message_builder.create_text_section_block( - "*Description*\n_No description_" - ) - ) - result = [] if self.status == "runtime error": result.extend( From 5650bcc74a8cdba5e737fbbb26bf02d40bf0b1c6 Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Wed, 27 Sep 2023 17:47:24 +0300 Subject: [PATCH 8/9] Revert "query new field" This reverts commit b47260178f15b8d8fee63f42a92c98b756a8a706. --- elementary/monitor/alerts/source_freshness.py | 1 - .../macros/alerts/get_source_freshness_alerts.sql | 6 ++---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/elementary/monitor/alerts/source_freshness.py b/elementary/monitor/alerts/source_freshness.py index 8cce6dc64..1155b807b 100644 --- a/elementary/monitor/alerts/source_freshness.py +++ b/elementary/monitor/alerts/source_freshness.py @@ -26,7 +26,6 @@ def __init__( filter: Optional[str], path: str, error: str, - freshness_description: Optional[str] = None, **kwargs, ) -> None: super().__init__(**kwargs) diff --git a/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql b/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql index 4804b06e0..77fb58bc2 100644 --- a/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql +++ b/elementary/monitor/dbt_project/macros/alerts/get_source_freshness_alerts.sql @@ -45,8 +45,7 @@ sources.meta as model_meta, sources.freshness_error_after, sources.freshness_warn_after, - sources.freshness_filter, - sources.freshness_description + sources.freshness_filter from alerts_in_time_limit left join sources on alerts_in_time_limit.unique_id = sources.unique_id ) @@ -86,8 +85,7 @@ 'tags': alert_dict.get('tags'), 'model_meta': alert_dict.get('model_meta'), 'suppression_status': alert_dict.get('suppression_status'), - 'sent_at': alert_dict.get('sent_at'), - 'freshness_description': alert_dict.get('freshness_description') + 'sent_at': alert_dict.get('sent_at') } %} {% do pending_alerts.append(pending_alert_dict) %} {% endfor %} From ad776b1a53aa9d46c7eb8e645aeda19da565cda4 Mon Sep 17 00:00:00 2001 From: Noy Arie Date: Thu, 28 Sep 2023 09:24:02 +0300 Subject: [PATCH 9/9] fix tests --- tests/mocks/fetchers/alerts_fetcher_mock.py | 35 ++++++++++++--------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/tests/mocks/fetchers/alerts_fetcher_mock.py b/tests/mocks/fetchers/alerts_fetcher_mock.py index ab5704115..3986d12c9 100644 --- a/tests/mocks/fetchers/alerts_fetcher_mock.py +++ b/tests/mocks/fetchers/alerts_fetcher_mock.py @@ -322,10 +322,11 @@ def query_pending_source_freshness_alerts(self, *args, **kwargs): schema_name="test_schema", source_name="source_1", identifier="identifier", - freshness_error_after="10", - freshness_warn_after="10", - freshness_filter="", + error_after="10", + warn_after="10", + filter="", status="fail", + normalized_status="fail", owners="[]", path="", error="", @@ -347,10 +348,11 @@ def query_pending_source_freshness_alerts(self, *args, **kwargs): schema_name="test_schema", source_name="source_1", identifier="identifier", - freshness_error_after="10", - freshness_warn_after="10", - freshness_filter="", + error_after="10", + warn_after="10", + filter="", status="fail", + normalized_status="fail", owners="[]", path="", error="", @@ -372,10 +374,11 @@ def query_pending_source_freshness_alerts(self, *args, **kwargs): schema_name="test_schema", source_name="source_2", identifier="identifier", - freshness_error_after="10", - freshness_warn_after="10", - freshness_filter="", + error_after="10", + warn_after="10", + filter="", status="fail", + normalized_status="fail", owners="[]", path="", error="", @@ -397,10 +400,11 @@ def query_pending_source_freshness_alerts(self, *args, **kwargs): schema_name="test_schema", source_name="source_3", identifier="identifier", - freshness_error_after="10", - freshness_warn_after="10", - freshness_filter="", + error_after="10", + warn_after="10", + filter="", status="fail", + normalized_status="fail", owners="[]", path="", error="", @@ -424,10 +428,11 @@ def query_pending_source_freshness_alerts(self, *args, **kwargs): schema_name="test_schema", source_name="source_3", identifier="identifier", - freshness_error_after="10", - freshness_warn_after="10", - freshness_filter="", + error_after="10", + warn_after="10", + filter="", status="fail", + normalized_status="fail", owners="[]", path="", error="",