From e23fbf9f09d9689fbce4a3b9f8a8d28feffbfd00 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 7 Aug 2024 11:14:58 +0330 Subject: [PATCH 1/3] feat: sending unique reports! --- dags/violation_detection_helpers/utils/prepare_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dags/violation_detection_helpers/utils/prepare_report.py b/dags/violation_detection_helpers/utils/prepare_report.py index 7d4c0c16..5f7e9d21 100644 --- a/dags/violation_detection_helpers/utils/prepare_report.py +++ b/dags/violation_detection_helpers/utils/prepare_report.py @@ -50,7 +50,7 @@ def prepare(self, transformed_documents: list[dict[str, Any]]) -> str | None: if len(reports) != 0: report = ( "Here's a list of messages with detected violation\n\n" - + "\n".join(reports) + + "\n".join(list(set(reports))) ) else: logging.warning( From 2a24a4bfaffbf2ed2c230367e0b7bd4eef1bbbc7 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 7 Aug 2024 11:22:33 +0330 Subject: [PATCH 2/3] fix: the ordering of tasks and XCOM! --- dags/discourse_analyzer_etl.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/dags/discourse_analyzer_etl.py b/dags/discourse_analyzer_etl.py index b590da8a..d209c841 100644 --- a/dags/discourse_analyzer_etl.py +++ b/dags/discourse_analyzer_etl.py @@ -204,4 +204,9 @@ def analyze_discourse(platform_processed: dict[str, str | bool]) -> None: raw_data_etl = discourse_etl_raw_data.expand(platform_info=platform_modules) raw_members_etl = discourse_etl_raw_members.expand(platform_info=platform_modules) - raw_members_etl >> analyze_discourse(platform_processed=raw_data_etl) + + analyze_discourse_task = analyze_discourse.expand( + platform_processed=platform_modules + ) + + [raw_data_etl, raw_members_etl] >> analyze_discourse_task From 870cd2173f273176d67711087dbb5270e9b82007 Mon Sep 17 00:00:00 2001 From: Mohammad Amin Date: Wed, 7 Aug 2024 11:34:38 +0330 Subject: [PATCH 3/3] fix: more flexible assertion! --- .../tests/unit/test_prepare_report.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dags/violation_detection_helpers/tests/unit/test_prepare_report.py b/dags/violation_detection_helpers/tests/unit/test_prepare_report.py index e47c20ab..31213d04 100644 --- a/dags/violation_detection_helpers/tests/unit/test_prepare_report.py +++ b/dags/violation_detection_helpers/tests/unit/test_prepare_report.py @@ -94,7 +94,8 @@ def test_multiple_valid_inputs(self): "Document link: https://sample_link2.com | Label: Identifying, Sexualized\n" "Document link: https://sample_link3.com | Label: Sexualized" ) - self.assertEqual(result, expected_report, "Expected a valid report string") + for report in expected_report.split("\n"): + self.assertIn(report, result, "Expected a valid report string") @patch("logging.error") def test_missing_link(self, mock_logging_error):