From 174972da82a50bfa93f6fdbdd77bb68f982780d4 Mon Sep 17 00:00:00 2001 From: Hui Song Date: Fri, 4 Oct 2024 09:18:43 -0400 Subject: [PATCH] remove owners and eda_deployment_type from collected data --- src/aap_eda/analytics/analytics_collectors.py | 47 ++++++++++++++----- .../analytics/test_analytics_collectors.py | 30 ++++-------- 2 files changed, 45 insertions(+), 32 deletions(-) diff --git a/src/aap_eda/analytics/analytics_collectors.py b/src/aap_eda/analytics/analytics_collectors.py index 9c5adc0dd..167c39888 100644 --- a/src/aap_eda/analytics/analytics_collectors.py +++ b/src/aap_eda/analytics/analytics_collectors.py @@ -20,7 +20,7 @@ from ansible_base.resource_registry.models.service_identifier import service_id from django.conf import settings from django.db import connection -from django.db.models import Manager, Q +from django.db.models import F, Manager, Q from insights_analytics_collector import CsvFileSplitter, register from aap_eda.core import models @@ -50,7 +50,6 @@ def config(**kwargs) -> dict: # skip license related info so far "eda_log_level": settings.APP_LOG_LEVEL, "eda_version": get_eda_version(), - "eda_deployment_type": settings.DEPLOYMENT_TYPE, } @@ -177,8 +176,35 @@ def decision_environments_table( def event_streams_table( since: datetime, full_path: str, until: datetime, **kwargs ): - query = _get_query(models.EventStream.objects, since, until) - return _copy_table("event_streams", query, full_path) + event_streams = ( + models.EventStream.objects.filter( + Q(created_at__gt=since, created_at__lte=until) + | Q(modified_at__gt=since, modified_at__lte=until) + ) + .order_by("id") + .distinct() + ).values( + "id", + "organization_id", + "name", + "event_stream_type", + "eda_credential_id", + "uuid", + "created_at", + "modified_at", + "events_received", + "last_event_received_at", + ) + + query = ( + str(event_streams.query) + .replace(_datetime_format(since), f"'{since.isoformat()}'") + .replace(_datetime_format(until), f"'{until.isoformat()}'") + ) + + return _copy_table( + "event_streams", f"COPY ({query}) TO STDOUT WITH CSV HEADER", full_path + ) @register( @@ -202,21 +228,18 @@ def event_streams_by_activation_table( if not bool(event_streams): return - event_streams = event_streams.extra( - select={ - "activation_id": "core_activation_event_streams.activation_id", - "event_stream_id": "core_eventstream.id", - } + event_streams = event_streams.annotate( + event_stream_id=F("id"), + activation_id=F("activations__id"), ).values( - "activation_id", - "event_stream_id", "name", "event_stream_type", "eda_credential_id", - "owner", "events_received", "last_event_received_at", "organization_id", + "event_stream_id", + "activation_id", ) query = f"COPY ({event_streams.query}) TO STDOUT WITH CSV HEADER" diff --git a/tests/integration/analytics/test_analytics_collectors.py b/tests/integration/analytics/test_analytics_collectors.py index 923f40583..28d27eb7b 100644 --- a/tests/integration/analytics/test_analytics_collectors.py +++ b/tests/integration/analytics/test_analytics_collectors.py @@ -65,14 +65,13 @@ def test_internal_infra_files(): data_collection_status_csv, encoding="utf-8" ) - assert len(config_json.keys()) == 5 + assert len(config_json.keys()) == 4 for key in config_json.keys(): assert key in [ "install_uuid", "platform", "eda_log_level", "eda_version", - "eda_deployment_type", ] assert manifest_json["config.json"] == "1.0" assert manifest_json["data_collection_status.csv"] == "1.0" @@ -404,15 +403,7 @@ def test_event_streams_table_collector( "name", "event_stream_type", "eda_credential_id", - "additional_data_headers", - "test_mode", - "test_content_type", - "test_content", - "test_headers", - "test_error_message", - "owner_id", "uuid", - "url", "created_at", "modified_at", "events_received", @@ -449,24 +440,23 @@ def test_event_streams_table_by_activation_collector( lines = list(reader) assert header == [ - "activation_id", - "event_stream_id", "name", "event_stream_type", "eda_credential_id", - "owner_id", "events_received", "last_event_received_at", "organization_id", + "event_stream_id", + "activation_id", ] assert len(lines) == 2 - assert lines[0][1] == str(default_event_streams[0].id) - assert lines[0][2] == default_event_streams[0].name - assert lines[0][3] == default_event_streams[0].event_stream_type - assert lines[1][1] == str(default_event_streams[1].id) - assert lines[1][2] == default_event_streams[1].name - assert lines[1][3] == default_event_streams[1].event_stream_type - assert sorted([lines[0][0], lines[1][0]]) == sorted( + assert lines[0][0] == default_event_streams[0].name + assert lines[0][1] == default_event_streams[0].event_stream_type + assert lines[0][6] == str(default_event_streams[0].id) + assert lines[1][0] == default_event_streams[1].name + assert lines[1][1] == default_event_streams[1].event_stream_type + assert lines[1][6] == str(default_event_streams[1].id) + assert sorted([lines[0][7], lines[1][7]]) == sorted( [str(default_activation.id), str(new_activation.id)] )