Collect job stats

src/aap_eda/analytics/analytics_collectors.py

@@ -9,6 +9,7 @@
 from django.db.models import Manager, Q
 from insights_analytics_collector import CsvFileSplitter, register
 
+from aap_eda.analytics.utils import extract_job_details
 from aap_eda.core import models
 from aap_eda.utils import get_eda_version
 
@@ -40,6 +41,40 @@ def config(**kwargs) -> dict:
  }
 
 
+@register(
+ "jobs_stats",
+ "1.0",
+ description="Stats data for jobs",
+)
+def jobs_stats(since: datetime, full_path: str, until: datetime, **kwargs):
+ stats = {}
+ audit_actions = _get_audit_action_qs(since, until)
+
+ if not bool(audit_actions):
+ return stats
+
+ for action in audit_actions.all():
+ job_type, job_id, install_uuid = extract_job_details(action.url)
+ if not job_type:
+ continue
+
+ data = stats.get(job_id, [])
+ job_stat = {}
+ job_stat["job_id"] = job_id
+ job_stat["type"] = job_type
+ job_stat["created_at"] = action.fired_at.strftime(
+ "%Y-%m-%dT%H:%M:%S.%fZ"
+ )
+ job_stat["status"] = action.status
+ job_stat["url"] = action.url
+ job_stat["install_uuid"] = install_uuid
+ data.append(job_stat)
+
+ stats[job_id] = data
+
+ return stats
+
+
 @register(
  "activations_table",
  "1.0",
@@ -344,7 +379,7 @@ def _get_audit_action_qs(since: datetime, until: datetime):
  else:
  audit_actions = models.AuditAction.objects.filter(
  audit_rule_id__in=tuple(audit_rule_ids)
- ).order_by("id")
+ ).order_by("fired_at")
 
  return audit_actions
 

src/aap_eda/analytics/utils.py

@@ -12,8 +12,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import base64
+import logging
+import re
+from typing import Optional, Tuple
+
+import requests
+import yaml
 from django.utils.dateparse import parse_datetime
 
+from aap_eda.core import enums, models
+from aap_eda.utils import str_to_bool
+
+logger = logging.getLogger("aap_eda.analytics")
+
 
 def datetime_hook(dt: dict) -> dict:
  new_dt = {}
@@ -23,3 +35,72 @@ def datetime_hook(dt: dict) -> dict:
  except TypeError:
  new_dt[key] = value
  return new_dt
+
+
+def collect_controllers_info() -> dict:
+ aap_credentia_type = models.CredentialType.objects.get(
+ name=enums.DefaultCredentialType.AAP
+ )
+ credentials = models.EdaCredential.objects.filter(
+ credential_type=aap_credentia_type
+ )
+ info = {}
+ for credential in credentials:
+ controller_info = {}
+ inputs = yaml.safe_load(credential.inputs.get_secret_value())
+ host = inputs["host"]
+ url = f"{host}/api/v2/ping/"
+ verify = str_to_bool(inputs.get("verify_ssl", ""))
+ token = inputs.get("oauth_token")
+
+ controller_info["credential_id"] = credential.id
+ controller_info["inputs"] = inputs
+ if token:
+ headers = {"Authorization": f"Bearer {token}"}
+ logger.info("Use Bearer token to ping the controller.")
+ else:
+ user_pass = f"{inputs.get('username')}:{inputs.get('password')}"
+ auth_value = (
+ f"Basic {base64.b64encode(user_pass.encode()).decode()}"
+ )
+ headers = {"Authorization": f"{auth_value}"}
+ logger.info("Use Basic authentication to ping the controller.")
+
+ try:
+ resp = requests.get(url, headers=headers, verify=verify)
+ resp_json = resp.json()
+ controller_info["install_uuid"] = resp_json["install_uuid"]
+
+ info[host] = controller_info
+ except requests.exceptions.RequestException as e:
+ logger.warning(
+ f"Credential {credential.name} failed in authorization: {e}"
+ )
+
+ return info
+
+
+def extract_job_details(
+ url: str,
+) -> Tuple[Optional[str], Optional[str], Optional[str]]:
+ controllers = collect_controllers_info()
+
+ for host, info in controllers.items():
+ if not url.startswith(host):
+ continue
+
+ pattern = r"/jobs/([a-zA-Z]+)/(\d+)/"
+
+ match = re.search(pattern, url)
+
+ if match:
+ job_type = match.group(1)
+ job_type = (
+ "run_job_template"
+ if job_type == "playbook"
+ else "run_workflow_template"
+ )
+ job_number = match.group(2)
+ return job_type, str(job_number), info["install_uuid"]
+
+ return None, None, None

tests/integration/analytics/test_analytics_collectors.py

@@ -17,7 +17,9 @@
 import os
 import tarfile
 import tempfile
+import uuid
 from datetime import timedelta
+from unittest.mock import patch
 
 import pytest
 from django.utils.timezone import now
@@ -89,11 +91,76 @@ def test_internal_infra_files():
  "status",
  "elapsed",
  ]
- assert len(lines) == 1
+ assert len(lines) == 2
 
  collector._gather_cleanup()
 
 
+@pytest.mark.django_db
+def test_jobs_stats_collector(
+ default_activation: models.Activation,
+ audit_action_1: models.AuditAction,
+ audit_action_2: models.AuditAction,
+ audit_action_3: models.AuditAction,
+):
+ until = now()
+ time_start = until - timedelta(hours=9)
+ job_ids = ["8018", "8020"]
+ intall_uuids = [str(uuid.uuid4()), str(uuid.uuid4())]
+ audit_action_1.url = (
+ f"https://controller_1/#/jobs/playbook/{job_ids[0]}/details/"
+ )
+ audit_action_2.url = (
+ f"https://controller_2/#/jobs/workflow/{job_ids[0]}/details/"
+ )
+ audit_action_3.url = (
+ f"https://controller_1/#/jobs/workflow/{job_ids[1]}/details/"
+ )
+ audit_action_1.save(update_fields=["url"])
+ audit_action_2.save(update_fields=["url"])
+ audit_action_3.save(update_fields=["url"])
+
+ with patch(
+ "aap_eda.analytics.utils.collect_controllers_info"
+ ) as collect_controllers_info:
+ collect_controllers_info.side_effect = [
+ {"https://controller_1/": {"install_uuid": intall_uuids[0]}},
+ {"https://controller_2/": {"install_uuid": intall_uuids[1]}},
+ {"https://controller_1/": {"install_uuid": intall_uuids[0]}},
+ ]
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ data = collectors.jobs_stats(
+ time_start, tmpdir, until=now() + timedelta(seconds=1)
+ )
+ assert list(data.keys()) == job_ids
+
+ first_jobs = data[job_ids[0]]
+ assert len(first_jobs) == 2
+ assert first_jobs[0]["job_id"] == job_ids[0]
+ assert first_jobs[0]["type"] == "run_job_template"
+ assert first_jobs[0]["created_at"] == audit_action_1.fired_at
+ assert first_jobs[0]["status"] == audit_action_1.status
+ assert first_jobs[0]["url"] == audit_action_1.url
+ assert first_jobs[0]["install_uuid"] == intall_uuids[0]
+
+ assert first_jobs[1]["job_id"] == job_ids[0]
+ assert first_jobs[1]["type"] == "run_workflow_template"
+ assert first_jobs[1]["created_at"] == audit_action_2.fired_at
+ assert first_jobs[1]["status"] == audit_action_2.status
+ assert first_jobs[1]["url"] == audit_action_2.url
+ assert first_jobs[1]["install_uuid"] == intall_uuids[1]
+
+ second_jobs = data[job_ids[1]]
+ assert len(second_jobs) == 1
+ assert second_jobs[0]["job_id"] == job_ids[1]
+ assert second_jobs[0]["type"] == "run_workflow_template"
+ assert second_jobs[0]["created_at"] == audit_action_3.fired_at
+ assert second_jobs[0]["status"] == audit_action_3.status
+ assert second_jobs[0]["url"] == audit_action_3.url
+ assert second_jobs[0]["install_uuid"] == intall_uuids[0]
+
+
 @pytest.mark.django_db
 def test_activations_table_collector(default_activation: models.Activation):
  until = now()

tests/integration/analytics/test_gather_analytics.py

@@ -83,32 +83,32 @@ def test_gather_analytics_invalid_settings(
  (
  ("--dry-run", "--since", "2024-08-20"),
  "INFO",
- "No analytics collected",
+ "Analytics collection is done",
  ),
  (
  ("--dry-run", "--since", "'2024-08-20 19:44:43.622759+00'"),
  "INFO",
- "No analytics collected",
+ "Analytics collection is done",
  ),
  (
  ("--dry-run", "--since", "'2024-08-20 19:44:43'"),
  "INFO",
- "No analytics collected",
+ "Analytics collection is done",
  ),
  (
  ("--dry-run", "--since", "'2024-08-20 19:44:43.622759'"),
  "INFO",
- "No analytics collected",
+ "Analytics collection is done",
  ),
  (
  ("--dry-run", "--until", "2024-09-20"),
  "INFO",
- "No analytics collected",
+ "Analytics collection is done",
  ),
  (
  ("--dry-run", "--until", "'2024-09-20 19:44:43.622759+00'"),
  "INFO",
- "No analytics collected",
+ "Analytics collection is done",
  ),
  (
  (
@@ -119,7 +119,7 @@ def test_gather_analytics_invalid_settings(
  "'2024-09-20 19:44:43'",
  ),
  "INFO",
- "No analytics collected",
+ "Analytics collection is done",
  ),
  ],
 )

tests/integration/analytics/test_utils.py

@@ -14,10 +14,12 @@
 
 import datetime
 import json
+import uuid
+from unittest.mock import patch
 
 from django.core.serializers.json import DjangoJSONEncoder
 
-from aap_eda.analytics.utils import datetime_hook
+from aap_eda.analytics.utils import datetime_hook, extract_job_details
 
 
 def test_datetime_hook():
@@ -44,3 +46,37 @@ def test_bad_datetime_hook():
 
  assert isinstance(result["started_at"], datetime.datetime) is True
  assert isinstance(result["ended_at"], datetime.datetime) is False
+
+
+def test_extract_job_details():
+ install_uuid = uuid.uuid4()
+ job_id = 8018
+
+ with patch(
+ "aap_eda.analytics.utils.collect_controllers_info"
+ ) as collect_controllers_info:
+ collect_controllers_info.return_value = {
+ "https://controller_1/": {"install_uuid": install_uuid}
+ }
+
+ job_type, job_id, retrieved_uuid = extract_job_details(
+ f"https://controller_1/#/jobs/workflow/{job_id}/details/"
+ )
+
+ assert job_type == "run_workflow_template"
+ assert job_id == "8018"
+ assert retrieved_uuid == install_uuid
+
+ job_type, job_id, retrieved_uuid = extract_job_details(
+ f"https://controller_1/#/jobs/playbook/{job_id}/details/"
+ )
+ assert job_type == "run_job_template"
+ assert job_id == "8018"
+ assert retrieved_uuid == install_uuid
+
+ job_type, job_id, retrieved_uuid = extract_job_details(
+ f"https://invalid_controller/#/jobs/workflow/{job_id}/details/"
+ )
+ assert job_type is None
+ assert job_id is None
+ assert retrieved_uuid is None