use python-user-agents to parse user agent string

if https://iplist.cc/api/ fails, the redirect will fail too, hence do this call in a celery task.
set enable_analytics - default=True
add visitor summary to shortened url admin

bots/admin.py

@@ -1,19 +1,15 @@
 import datetime
-import json
 
 import django.db.models
 from django import forms
 from django.conf import settings
 from django.contrib import admin
-from django.db import DataError
-from django.db.models import Max, Count, F, Func
+from django.db.models import Max, Count, F
 from django.http import HttpResponse
 from django.template import loader
-from django.urls import reverse
 from django.utils import dateformat
 from django.utils.safestring import mark_safe
 from django.utils.timesince import timesince
-from furl import furl
 
 from bots.admin_links import list_related_html_url, open_in_new_tab, change_obj_url
 from bots.models import (
@@ -27,6 +23,9 @@
     BotIntegration,
 )
 from bots.tasks import create_personal_channels_for_all_members
+from gooeysite.custom_filters import (
+    related_json_field_summary,
+)
 from gooeysite.custom_widgets import JSONEditorWidget
 
 
@@ -221,55 +220,13 @@ def view_analysis_results(self, bi: BotIntegration):
         ).exclude(
             analysis_result={},
         )
-        max_depth = 3
-        field = "analysis_result"
-        nested_keys = [field]
-        for i in range(max_depth):
-            next_keys = []
-            for parent in nested_keys:
-                try:
-                    next_keys.extend(
-                        f"{parent}__{child}"
-                        for child in (
-                            msgs.values(parent)
-                            .annotate(
-                                keys=Func(F(parent), function="jsonb_object_keys")
-                            )
-                            .order_by()
-                            .distinct()
-                            .values_list("keys", flat=True)
-                        )
-                    )
-                except DataError:
-                    next_keys.append(parent)
-            nested_keys = next_keys
-        results = {
-            key.split(field + "__")[-1]: [
-                (
-                    json.dumps(val).strip('"'),
-                    count,
-                    furl(
-                        reverse(
-                            f"admin:{Message._meta.app_label}_{Message.__name__.lower()}_changelist"
-                        ),
-                        query_params={
-                            f"conversation__bot_integration__id__exact": bi.id,
-                            key: val,
-                        },
-                    ),
-                )
-                for val, count in (
-                    msgs.values(key)
-                    .annotate(count=Count("id"))
-                    .order_by("-count")
-                    .values_list(key, "count")
-                )
-                if val is not None
-            ]
-            for key in nested_keys
-        }
-        if not results:
-            raise Message.DoesNotExist
+        results = related_json_field_summary(
+            Message.objects,
+            "analysis_result",
+            qs=msgs,
+            query_param="conversation__bot_integration__id__exact",
+            instance_id=bi.id,
+        )
         html = loader.render_to_string(
             "anaylsis_result.html", context=dict(results=results)
         )

gooeysite/custom_filters.py

@@ -0,0 +1,84 @@
+import json
+
+from django.db import DataError
+from django.db.models import F, Func, QuerySet, Count
+from django.urls import reverse
+from furl import furl
+
+
+def json_field_nested_lookup_keys(
+    qs: QuerySet, field: str, max_depth: int = 3
+) -> list[str]:
+    nested_keys = [field]
+    for _ in range(max_depth):
+        next_keys = []
+        for parent in nested_keys:
+            try:
+                next_keys.extend(
+                    f"{parent}__{child}"
+                    for child in (
+                        qs.values(parent)
+                        .annotate(keys=Func(F(parent), function="jsonb_object_keys"))
+                        .order_by()
+                        .distinct()
+                        .values_list("keys", flat=True)
+                    )
+                )
+            except DataError:
+                next_keys.append(parent)
+        nested_keys = next_keys
+    return nested_keys
+
+
+def related_json_field_summary(
+    manager, field, qs=None, query_param=None, instance_id=None
+):
+    if query_param is None:
+        try:
+            query_field_name = manager.field.name
+        except AttributeError:
+            query_field_name = manager.query_field_name
+        query_param = f"{query_field_name}__id__exact"
+
+    model = manager.model
+    meta = model._meta
+
+    if instance_id is None:
+        instance_id = manager.instance.id
+    if instance_id is None:
+        raise model.DoesNotExist
+    instance_id = str(instance_id)
+
+    if qs is None:
+        qs = manager.all()
+
+    nested_keys = json_field_nested_lookup_keys(qs, field)
+
+    results = {
+        key.split(field + "__")[-1]: [
+            (
+                json.dumps(val).strip('"'),
+                count,
+                furl(
+                    reverse(
+                        f"admin:{meta.app_label}_{model.__name__.lower()}_changelist"
+                    ),
+                    query_params={
+                        query_param: instance_id,
+                        field: json.dumps([key, val]),
+                    },
+                ),
+            )
+            for val, count in (
+                qs.values(key)
+                .annotate(count=Count("id"))
+                .order_by("-count")
+                .values_list(key, "count")
+            )
+            if val is not None
+        ]
+        for key in nested_keys
+    }
+    if not results:
+        raise model.DoesNotExist
+    return results

pyproject.toml

@@ -75,6 +75,9 @@ tabulate = "^0.9.0"
 deepgram-sdk = "^2.11.0"
 scipy = "^1.11.2"
 rank-bm25 = "^0.2.2"
+pyyaml = "^6.0.1"
+ua-parser = "^0.18.0"
+user-agents = "^2.2.0"
 
 [tool.poetry.group.dev.dependencies]
 watchdog = "^2.1.9"