From 4e73cb29a4937b1c52aa0fc163bfd24c34601f38 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Wed, 3 Apr 2024 09:25:43 -0500
Subject: [PATCH 01/30] add owners list to models

---
 backend/app/heartbeat_listener_sync.py |  7 +++++++
 backend/app/models/listeners.py        | 12 +++++++++++-
 backend/app/tests/test_extractors.py   | 13 ++++++++++++-
 backend/app/tests/utils.py             |  6 +++++-
 4 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/backend/app/heartbeat_listener_sync.py b/backend/app/heartbeat_listener_sync.py
index f975b53c5..c91d8804f 100644
--- a/backend/app/heartbeat_listener_sync.py
+++ b/backend/app/heartbeat_listener_sync.py
@@ -26,6 +26,9 @@ def callback(ch, method, properties, body):
     extractor_db = EventListenerDB(
         **extractor_info, properties=ExtractorInfo(**extractor_info)
     )
+    owner = msg["owner"]
+    if owner is not None:
+        extractor_db.owners = {"users": [owner]}
 
     mongo_client = MongoClient(settings.MONGODB_URL)
     db = mongo_client[settings.MONGO_DATABASE]
@@ -33,6 +36,10 @@ def callback(ch, method, properties, body):
     # check to see if extractor alredy exists
     existing_extractor = db["listeners"].find_one({"name": msg["queue"]})
     if existing_extractor is not None:
+        if owner is not None:
+            # TODO: make sure owner is included in owners list
+            pass
+
         # Update existing listener
         existing_version = existing_extractor["version"]
         new_version = extractor_db.version
diff --git a/backend/app/models/listeners.py b/backend/app/models/listeners.py
index 80b388c49..4d0d518ef 100644
--- a/backend/app/models/listeners.py
+++ b/backend/app/models/listeners.py
@@ -38,6 +38,15 @@ class ExtractorInfo(BaseModel):
     categories: Optional[List[str]] = []
     parameters: Optional[dict] = None
     version: Optional[str] = "1.0"
+    unique_key: Optional[str] = None
+
+
+class OwnerList(BaseModel):
+    """Container object for lists of user emails/group IDs/dataset IDs that can submit to listener."""
+
+    users: List[str] = []
+    group: List[PydanticObjectId] = []
+    datasets: List[PydanticObjectId] = []
 
 
 class EventListenerBase(BaseModel):
@@ -46,6 +55,7 @@ class EventListenerBase(BaseModel):
     name: str
     version: str = "1.0"
     description: str = ""
+    owners: Optional[OwnerList] = None
 
 
 class EventListenerIn(EventListenerBase):
@@ -69,7 +79,7 @@ class EventListenerDB(Document, EventListenerBase):
     created: datetime = Field(default_factory=datetime.now)
     modified: datetime = Field(default_factory=datetime.now)
     lastAlive: datetime = None
-    alive: Optional[bool] = None  # made up field to indicate if extractor is alive
+    alive: Optional[bool] = None
     properties: Optional[ExtractorInfo] = None
 
     class Settings:
diff --git a/backend/app/tests/test_extractors.py b/backend/app/tests/test_extractors.py
index b747ed264..abe237fc1 100644
--- a/backend/app/tests/test_extractors.py
+++ b/backend/app/tests/test_extractors.py
@@ -1,7 +1,12 @@
 from fastapi.testclient import TestClient
 
 from app.config import settings
-from app.tests.utils import create_dataset, upload_file, register_v1_extractor
+from app.tests.utils import (
+    create_user,
+    create_dataset,
+    upload_file,
+    register_v1_extractor,
+)
 
 
 def test_register(client: TestClient, headers: dict):
@@ -9,6 +14,12 @@ def test_register(client: TestClient, headers: dict):
     register_v1_extractor(client, headers, ext_name)
 
 
+def test_register_private(client: TestClient, headers: dict):
+    ext_name = "test.test_register_private"
+    create_user(client, headers, email="extract_master@email.com")
+    register_v1_extractor(client, headers, ext_name, user="extract_master@email.com")
+
+
 def test_get_one(client: TestClient, headers: dict):
     extractor_id = register_v1_extractor(client, headers).get("id")
     response = client.get(
diff --git a/backend/app/tests/utils.py b/backend/app/tests/utils.py
index a81fc1730..47261987e 100644
--- a/backend/app/tests/utils.py
+++ b/backend/app/tests/utils.py
@@ -221,11 +221,15 @@ def create_folder(
     return response.json()
 
 
-def register_v1_extractor(client: TestClient, headers: dict, name: str = None):
+def register_v1_extractor(
+    client: TestClient, headers: dict, name: str = None, user: str = None
+):
     """Registers a new v1 listener (extractor) and returns the JSON."""
     new_extractor = extractor_info_v1_example
     if name:
         new_extractor["name"] = name
+    if user:
+        new_extractor["owners"] = {"users": [user]}
     response = client.post(
         f"{settings.API_V2_STR}/extractors", json=new_extractor, headers=headers
     )

From de2e0379e01048e04693c3fb0a38a3e709a338cf Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Fri, 5 Apr 2024 09:14:51 -0500
Subject: [PATCH 02/30] add owner check to lookup

---
 backend/app/heartbeat_listener_sync.py | 17 ++++++++++-------
 backend/app/models/listeners.py        |  4 +++-
 2 files changed, 13 insertions(+), 8 deletions(-)

diff --git a/backend/app/heartbeat_listener_sync.py b/backend/app/heartbeat_listener_sync.py
index c91d8804f..3c07d7e31 100644
--- a/backend/app/heartbeat_listener_sync.py
+++ b/backend/app/heartbeat_listener_sync.py
@@ -28,18 +28,21 @@ def callback(ch, method, properties, body):
     )
     owner = msg["owner"]
     if owner is not None:
-        extractor_db.owners = {"users": [owner]}
+        extractor_db.owners = {"owner": owner}
 
     mongo_client = MongoClient(settings.MONGODB_URL)
     db = mongo_client[settings.MONGO_DATABASE]
 
-    # check to see if extractor alredy exists
-    existing_extractor = db["listeners"].find_one({"name": msg["queue"]})
+    # check to see if extractor already exists
+    if owner is not None:
+        existing_extractor = EventListenerDB.find_one(
+            EventListenerDB.name == msg["queue"]
+        )
+    else:
+        existing_extractor = EventListenerDB.find_one(
+            EventListenerDB.name == msg["queue"], EventListenerDB.owners.owner == owner
+        )
     if existing_extractor is not None:
-        if owner is not None:
-            # TODO: make sure owner is included in owners list
-            pass
-
         # Update existing listener
         existing_version = existing_extractor["version"]
         new_version = extractor_db.version
diff --git a/backend/app/models/listeners.py b/backend/app/models/listeners.py
index 4d0d518ef..9453d20d8 100644
--- a/backend/app/models/listeners.py
+++ b/backend/app/models/listeners.py
@@ -42,8 +42,10 @@ class ExtractorInfo(BaseModel):
 
 
 class OwnerList(BaseModel):
-    """Container object for lists of user emails/group IDs/dataset IDs that can submit to listener."""
+    """Container object for lists of user emails/group IDs/dataset IDs that can submit to listener.
+    The singular owner is the primary who can modify other lists."""
 
+    owner: str
     users: List[str] = []
     group: List[PydanticObjectId] = []
     datasets: List[PydanticObjectId] = []

From 1ce7bf27c2dfb2cf3034c6cc02ed7ff0b3ab52f0 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Fri, 5 Apr 2024 10:23:31 -0500
Subject: [PATCH 03/30] add logic to filter inaccessible extractors

---
 backend/app/heartbeat_listener_sync.py | 15 +++---
 backend/app/models/listeners.py        |  7 +--
 backend/app/routers/authorization.py   |  2 +-
 backend/app/routers/groups.py          |  4 +-
 backend/app/routers/listeners.py       | 66 +++++++++++++++++++++-----
 5 files changed, 69 insertions(+), 25 deletions(-)

diff --git a/backend/app/heartbeat_listener_sync.py b/backend/app/heartbeat_listener_sync.py
index 5908c0339..bce1aca16 100644
--- a/backend/app/heartbeat_listener_sync.py
+++ b/backend/app/heartbeat_listener_sync.py
@@ -1,7 +1,8 @@
 import json
 import logging
-
 import pika
+from beanie import PydanticObjectId
+
 from app.config import settings
 from app.models.listeners import EventListenerDB, EventListenerOut, ExtractorInfo
 from app.models.search import SearchCriteria
@@ -48,10 +49,10 @@ def callback(ch, method, properties, body):
         new_version = extractor_db.version
         if version.parse(new_version) > version.parse(existing_version):
             # if this is a new version, add it to the database
-            new_extractor = db["listeners"].insert_one(extractor_db.to_mongo())
-            found = db["listeners"].find_one({"_id": new_extractor.inserted_id})
+            new_extractor = EventListenerDB.insert_one(extractor_db.to_mongo())
+            found = EventListenerDB.get(PydanticObjectId(new_extractor.inserted_id))
             # TODO - for now we are not deleting an older version of the extractor, just adding a new one
-            # removed = db["listeners"].delete_one({"_id": existing_extractor["_id"]})
+            # removed = EventListenerDB.delete_one(EventListenerDB.id == PydanticObjectId(existing_extractor["_id"]))
             extractor_out = EventListenerOut.from_mongo(found)
             logger.info(
                 "%s updated from %s to %s"
@@ -60,8 +61,8 @@ def callback(ch, method, properties, body):
             return extractor_out
     else:
         # Register new listener
-        new_extractor = db["listeners"].insert_one(extractor_db.to_mongo())
-        found = db["listeners"].find_one({"_id": new_extractor.inserted_id})
+        new_extractor = EventListenerDB.insert_one(extractor_db.to_mongo())
+        found = EventListenerDB.get(PydanticObjectId(new_extractor.inserted_id))
         extractor_out = EventListenerOut.from_mongo(found)
         logger.info("New extractor registered: " + extractor_name)
 
@@ -104,7 +105,7 @@ def callback(ch, method, properties, body):
                         FeedListener(listener_id=extractor_out.id, automatic=True)
                     ],
                 )
-                db["feeds"].insert_one(new_feed.to_mongo())
+                FeedDB.insert_one(new_feed.to_mongo())
 
         return extractor_out
 
diff --git a/backend/app/models/listeners.py b/backend/app/models/listeners.py
index a53d4bebe..9f8b46bfb 100644
--- a/backend/app/models/listeners.py
+++ b/backend/app/models/listeners.py
@@ -3,12 +3,13 @@
 from typing import List, Optional, Union
 
 import pymongo
+from beanie import Document, PydanticObjectId, View
+from pydantic import AnyUrl, BaseModel, Field
+
 from app.config import settings
 from app.models.authorization import AuthorizationDB
 from app.models.mongomodel import MongoDBRef
 from app.models.users import UserOut
-from beanie import Document, PydanticObjectId, View
-from pydantic import AnyUrl, BaseModel, Field
 
 
 class Repository(BaseModel):
@@ -46,7 +47,7 @@ class OwnerList(BaseModel):
 
     owner: str
     users: List[str] = []
-    group: List[PydanticObjectId] = []
+    groups: List[PydanticObjectId] = []
     datasets: List[PydanticObjectId] = []
 
 
diff --git a/backend/app/routers/authorization.py b/backend/app/routers/authorization.py
index 4aa1c46f0..51129c8c4 100644
--- a/backend/app/routers/authorization.py
+++ b/backend/app/routers/authorization.py
@@ -74,7 +74,7 @@ async def get_dataset_role(
     """Retrieve role of user for a specific dataset."""
     # Get group id and the associated users from authorization
     criteria = []
-    if not admin or not admin_mode:
+    if not admin and not admin_mode:
         criteria.append(
             Or(
                 AuthorizationDB.creator == current_user,
diff --git a/backend/app/routers/groups.py b/backend/app/routers/groups.py
index 1434d806a..f807f194e 100644
--- a/backend/app/routers/groups.py
+++ b/backend/app/routers/groups.py
@@ -46,7 +46,7 @@ async def get_groups(
 
     """
     criteria_list = []
-    if not admin or not admin_mode:
+    if not admin and not admin_mode:
         criteria_list.append(
             Or(
                 GroupDB.creator == user_id,
@@ -96,7 +96,7 @@ async def search_group(
             RegEx(field=GroupDB.description, pattern=search_term),
         ),
     ]
-    if not admin or not admin_mode:
+    if not admin and not admin_mode:
         criteria_list.append(
             Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id)
         )
diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index 47df813b8..611e0d3ad 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -4,10 +4,17 @@
 import string
 from typing import List, Optional
 
+from beanie import PydanticObjectId
+from beanie.operators import Or, RegEx, In, Exists
+from bson import ObjectId
+from fastapi import APIRouter, Depends, HTTPException
+from packaging import version
+
 from app.config import settings
 from app.keycloak_auth import get_current_user, get_current_username, get_user
 from app.models.config import ConfigEntryDB
 from app.models.feeds import FeedDB, FeedListener
+from app.models.groups import GroupDB
 from app.models.listeners import (
     EventListenerDB,
     EventListenerIn,
@@ -18,12 +25,8 @@
 from app.models.pages import Paged, _construct_page_metadata, _get_page_query
 from app.models.search import SearchCriteria
 from app.models.users import UserOut
+from app.routers.authentication import get_admin, get_admin_mode
 from app.routers.feeds import disassociate_listener_db
-from beanie import PydanticObjectId
-from beanie.operators import Or, RegEx
-from bson import ObjectId
-from fastapi import APIRouter, Depends, HTTPException
-from packaging import version
 
 router = APIRouter()
 legacy_router = APIRouter()  # for back-compatibilty with v1 extractors
@@ -196,7 +199,9 @@ async def search_listeners(
     skip: int = 0,
     limit: int = 2,
     heartbeat_interval: Optional[int] = settings.listener_heartbeat_interval,
-    user=Depends(get_current_username),
+    user_id=Depends(get_current_username),
+    admin_mode: bool = Depends(get_admin_mode),
+    admin=Depends(get_admin),
 ):
     """Search all Event Listeners in the db based on text.
 
@@ -211,13 +216,29 @@ async def search_listeners(
         _get_page_query(skip, limit, sort_field="name", ascending=True),
     ]
 
-    listeners_and_count = (
-        await EventListenerDB.find(
+    criteria_list = [
+        Or(
+            RegEx(field=EventListenerDB.name, pattern=text),
+            RegEx(field=EventListenerDB.description, pattern=text),
+        )
+    ]
+    if not admin and not admin_mode:
+        user_q = await GroupDB.find(
+            Or(GroupDB.creator == user_id, GroupDB.users.user.email == user),
+        ).to_list()
+        user_groups = [u["_id"] for u in user_q]
+
+        criteria_list.append(
             Or(
-                RegEx(field=EventListenerDB.name, pattern=text),
-                RegEx(field=EventListenerDB.description, pattern=text),
-            ),
+                Exists(EventListenerDB.owners, False),
+                EventListenerDB.owners.owner == user_id,
+                EventListenerDB.owners.users == user_id,
+                In(EventListenerDB.owners.groups, user_groups),
+            )
         )
+
+    listeners_and_count = (
+        await EventListenerDB.find(*criteria_list)
         .aggregate(aggregation_pipeline)
         .to_list()
     )
@@ -277,6 +298,8 @@ async def get_listeners(
     category: Optional[str] = None,
     label: Optional[str] = None,
     alive_only: Optional[bool] = False,
+    admin_mode: bool = Depends(get_admin_mode),
+    admin=Depends(get_admin),
 ):
     """Get a list of all Event Listeners in the db.
 
@@ -306,10 +329,29 @@ async def get_listeners(
         _get_page_query(skip, limit, sort_field="name", ascending=True)
     )
 
+    # Filter by ownership
+    criteria_list = []
+    if not admin and not admin_mode:
+        user_q = await GroupDB.find(
+            Or(GroupDB.creator == user_id, GroupDB.users.user.email == user),
+        ).to_list()
+        user_groups = [u["_id"] for u in user_q]
+
+        criteria_list.append(
+            Or(
+                Exists(EventListenerDB.owners, False),
+                EventListenerDB.owners.owner == user_id,
+                EventListenerDB.owners.users == user_id,
+                In(EventListenerDB.owners.groups, user_groups),
+            )
+        )
+
     # Run aggregate query and return
     # Sort by name alphabetically
     listeners_and_count = (
-        await EventListenerDB.find().aggregate(aggregation_pipeline).to_list()
+        await EventListenerDB.find(*criteria_list)
+        .aggregate(aggregation_pipeline)
+        .to_list()
     )
     page_metadata = _construct_page_metadata(listeners_and_count, skip, limit)
     page = Paged(

From 02600f197bc0389701aa3f5bba8d8878f6142646 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Fri, 5 Apr 2024 10:34:29 -0500
Subject: [PATCH 04/30] add permission checks to edit endpoints

---
 backend/app/routers/listeners.py | 31 +++++++++++++++++++++++--------
 1 file changed, 23 insertions(+), 8 deletions(-)

diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index 611e0d3ad..c7a5e34d1 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -369,6 +369,8 @@ async def edit_listener(
     listener_id: str,
     listener_in: EventListenerIn,
     user_id=Depends(get_user),
+    admin_mode: bool = Depends(get_admin_mode),
+    admin=Depends(get_admin),
 ):
     """Update the information about an existing Event Listener..
 
@@ -376,11 +378,16 @@ async def edit_listener(
         listener_id -- UUID of the listener to be udpated
         listener_in -- JSON object including updated information
     """
-    listener = await EventListenerDB.find_one(
-        EventListenerDB.id == ObjectId(listener_id)
-    )
+    criteria_list = [EventListenerDB.id == ObjectId(listener_id)]
+    if not admin and not admin_mode:
+        criteria_list.append(
+            Or(
+                Exists(EventListenerDB.owners, False),
+                EventListenerDB.owners.owner == user_id,
+            ),
+        )
+    listener = await EventListenerDB.find_one(*criteria_list)
     if listener:
-        # TODO: Refactor this with permissions checks etc.
         listener_update = dict(listener_in) if listener_in is not None else {}
         listener_update["modified"] = datetime.datetime.utcnow()
         try:
@@ -395,12 +402,20 @@ async def edit_listener(
 @router.delete("/{listener_id}")
 async def delete_listener(
     listener_id: str,
-    user=Depends(get_current_username),
+    user_id=Depends(get_current_username),
+    admin_mode: bool = Depends(get_admin_mode),
+    admin=Depends(get_admin),
 ):
     """Remove an Event Listener from the database. Will not clear event history for the listener."""
-    listener = await EventListenerDB.find_one(
-        EventListenerDB.id == ObjectId(listener_id)
-    )
+    criteria_list = [EventListenerDB.id == ObjectId(listener_id)]
+    if not admin and not admin_mode:
+        criteria_list.append(
+            Or(
+                Exists(EventListenerDB.owners, False),
+                EventListenerDB.owners.owner == user_id,
+            ),
+        )
+    listener = await EventListenerDB.find_one(*criteria_list)
     if listener:
         # unsubscribe the listener from any feeds
         async for feed in FeedDB.find(

From 76203575e3f323c205c7b7b4f8e7747b4fd923c9 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Fri, 5 Apr 2024 10:50:25 -0500
Subject: [PATCH 05/30] new endpoints, rename Owners to Access

---
 backend/app/heartbeat_listener_sync.py |  4 +-
 backend/app/models/listeners.py        |  4 +-
 backend/app/routers/listeners.py       | 72 +++++++++++++++++++++-----
 3 files changed, 64 insertions(+), 16 deletions(-)

diff --git a/backend/app/heartbeat_listener_sync.py b/backend/app/heartbeat_listener_sync.py
index bce1aca16..972a85353 100644
--- a/backend/app/heartbeat_listener_sync.py
+++ b/backend/app/heartbeat_listener_sync.py
@@ -29,7 +29,7 @@ def callback(ch, method, properties, body):
     )
     owner = msg["owner"]
     if owner is not None:
-        extractor_db.owners = {"owner": owner}
+        extractor_db.access = {"owner": owner}
 
     mongo_client = MongoClient(settings.MONGODB_URL)
     db = mongo_client[settings.MONGO_DATABASE]
@@ -41,7 +41,7 @@ def callback(ch, method, properties, body):
         )
     else:
         existing_extractor = EventListenerDB.find_one(
-            EventListenerDB.name == msg["queue"], EventListenerDB.owners.owner == owner
+            EventListenerDB.name == msg["queue"], EventListenerDB.access.owner == owner
         )
     if existing_extractor is not None:
         # Update existing listener
diff --git a/backend/app/models/listeners.py b/backend/app/models/listeners.py
index 9f8b46bfb..1b11a9088 100644
--- a/backend/app/models/listeners.py
+++ b/backend/app/models/listeners.py
@@ -41,7 +41,7 @@ class ExtractorInfo(BaseModel):
     unique_key: Optional[str] = None
 
 
-class OwnerList(BaseModel):
+class AccessList(BaseModel):
     """Container object for lists of user emails/group IDs/dataset IDs that can submit to listener.
     The singular owner is the primary who can modify other lists."""
 
@@ -57,7 +57,7 @@ class EventListenerBase(BaseModel):
     name: str
     version: str = "1.0"
     description: str = ""
-    owners: Optional[OwnerList] = None
+    access: Optional[AccessList] = None
 
 
 class EventListenerIn(EventListenerBase):
diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index c7a5e34d1..74116fcde 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -230,10 +230,10 @@ async def search_listeners(
 
         criteria_list.append(
             Or(
-                Exists(EventListenerDB.owners, False),
-                EventListenerDB.owners.owner == user_id,
-                EventListenerDB.owners.users == user_id,
-                In(EventListenerDB.owners.groups, user_groups),
+                Exists(EventListenerDB.access, False),
+                EventListenerDB.access.owner == user_id,
+                EventListenerDB.access.users == user_id,
+                In(EventListenerDB.access.groups, user_groups),
             )
         )
 
@@ -339,10 +339,10 @@ async def get_listeners(
 
         criteria_list.append(
             Or(
-                Exists(EventListenerDB.owners, False),
-                EventListenerDB.owners.owner == user_id,
-                EventListenerDB.owners.users == user_id,
-                In(EventListenerDB.owners.groups, user_groups),
+                Exists(EventListenerDB.access, False),
+                EventListenerDB.access.owner == user_id,
+                EventListenerDB.access.users == user_id,
+                In(EventListenerDB.access.groups, user_groups),
             )
         )
 
@@ -382,8 +382,8 @@ async def edit_listener(
     if not admin and not admin_mode:
         criteria_list.append(
             Or(
-                Exists(EventListenerDB.owners, False),
-                EventListenerDB.owners.owner == user_id,
+                Exists(EventListenerDB.access, False),
+                EventListenerDB.access.owner == user_id,
             ),
         )
     listener = await EventListenerDB.find_one(*criteria_list)
@@ -411,8 +411,8 @@ async def delete_listener(
     if not admin and not admin_mode:
         criteria_list.append(
             Or(
-                Exists(EventListenerDB.owners, False),
-                EventListenerDB.owners.owner == user_id,
+                Exists(EventListenerDB.access, False),
+                EventListenerDB.access.owner == user_id,
             ),
         )
     listener = await EventListenerDB.find_one(*criteria_list)
@@ -425,3 +425,51 @@ async def delete_listener(
         await listener.delete()
         return {"deleted": listener_id}
     raise HTTPException(status_code=404, detail=f"Listener {listener_id} not found")
+
+
+@router.post("/{listener_id}/users/{target_user}")
+async def add_user_permission(
+    listener_id: str,
+    target_user: str,
+    user_id=Depends(get_current_username),
+    admin_mode: bool = Depends(get_admin_mode),
+    admin=Depends(get_admin),
+):
+    criteria_list = [EventListenerDB.id == ObjectId(listener_id)]
+    if not admin and not admin_mode:
+        criteria_list.append(
+            Or(
+                Exists(EventListenerDB.access, False),
+                EventListenerDB.access.owner == user_id,
+            ),
+        )
+    listener = await EventListenerDB.find_one(*criteria_list)
+    if listener:
+        if target_user not in listener.access.users:
+            listener.access.users.append(target_user)
+            await listener.save()
+        return listener.dict()
+    raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")
+
+
+@router.delete("/{listener_id}/users/{target_user}")
+async def remove_user_permission(
+    listener_id: str,
+    target_user: str,
+    user_id=Depends(get_current_username),
+    admin_mode: bool = Depends(get_admin_mode),
+    admin=Depends(get_admin),
+):
+    criteria_list = [EventListenerDB.id == ObjectId(listener_id)]
+    if not admin and not admin_mode:
+        criteria_list.append(
+            Or(
+                Exists(EventListenerDB.access, False),
+                EventListenerDB.access.owner == user_id,
+            ),
+        )
+    listener = await EventListenerDB.find_one(*criteria_list)
+    if listener:
+        listener.access.users.remove(target_user)
+        await listener.save()
+    raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")

From 44481f71a445e713e0af1b0698499d9a983bd158 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Fri, 12 Apr 2024 11:57:12 -0500
Subject: [PATCH 06/30] Update heartbeat_listener.py

---
 backend/heartbeat_listener.py | 18 ++++++++++++++----
 1 file changed, 14 insertions(+), 4 deletions(-)

diff --git a/backend/heartbeat_listener.py b/backend/heartbeat_listener.py
index c26c85d50..5c1488604 100644
--- a/backend/heartbeat_listener.py
+++ b/backend/heartbeat_listener.py
@@ -7,11 +7,12 @@
 
 from aio_pika import connect_robust
 from aio_pika.abc import AbstractIncomingMessage
+from packaging import version
+
 from app.config import settings
 from app.main import startup_beanie
 from app.models.listeners import EventListenerDB, EventListenerOut, ExtractorInfo
 from app.routers.listeners import _process_incoming_v1_extractor_info
-from packaging import version
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -34,11 +35,20 @@ async def callback(message: AbstractIncomingMessage):
         extractor_db = EventListenerDB(
             **extractor_info, properties=ExtractorInfo(**extractor_info)
         )
+        owner = msg["owner"]
+        if owner is not None:
+            extractor_db.access = {"owner": owner}
 
         # check to see if extractor already exists and update if so
-        existing_extractor = await EventListenerDB.find_one(
-            EventListenerDB.name == msg["queue"]
-        )
+        if owner is not None:
+            existing_extractor = await EventListenerDB.find_one(
+                EventListenerDB.name == msg["queue"]
+            )
+        else:
+            existing_extractor = await EventListenerDB.find_one(
+                EventListenerDB.name == msg["queue"],
+                EventListenerDB.access.owner == owner,
+            )
         if existing_extractor is not None:
             extractor_db.id = existing_extractor.id
             extractor_db.created = existing_extractor.created

From c125e49d662c70e9da26422a706b1e3b8aeda405 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Mon, 15 Apr 2024 09:27:41 -0500
Subject: [PATCH 07/30] update heartbeat logic

---
 backend/app/routers/listeners.py |  2 +-
 backend/heartbeat_listener.py    | 30 +++++++++++++++++++-----------
 2 files changed, 20 insertions(+), 12 deletions(-)

diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index 74116fcde..279e114ee 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -333,7 +333,7 @@ async def get_listeners(
     criteria_list = []
     if not admin and not admin_mode:
         user_q = await GroupDB.find(
-            Or(GroupDB.creator == user_id, GroupDB.users.user.email == user),
+            Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id),
         ).to_list()
         user_groups = [u["_id"] for u in user_q]
 
diff --git a/backend/heartbeat_listener.py b/backend/heartbeat_listener.py
index 5c1488604..de52b3601 100644
--- a/backend/heartbeat_listener.py
+++ b/backend/heartbeat_listener.py
@@ -31,24 +31,32 @@ async def callback(message: AbstractIncomingMessage):
         msg = json.loads(message.body.decode("utf-8"))
 
         extractor_info = msg["extractor_info"]
-        extractor_name = extractor_info["name"]
-        extractor_db = EventListenerDB(
-            **extractor_info, properties=ExtractorInfo(**extractor_info)
-        )
         owner = msg["owner"]
         if owner is not None:
-            extractor_db.access = {"owner": owner}
-
-        # check to see if extractor already exists and update if so
-        if owner is not None:
+            # Extractor name should match queue, which includes secret key with common extractor_info["name"]
+            extractor_name = msg["queue"]
+            extractor_db = EventListenerDB(
+                **extractor_info,
+                extractor_name=extractor_name,
+                access={"owner": owner},
+                properties=ExtractorInfo(**extractor_info),
+            )
+            logger.info(f"Received heartbeat from ${extractor_name} owned by ${owner}")
             existing_extractor = await EventListenerDB.find_one(
-                EventListenerDB.name == msg["queue"]
+                EventListenerDB.name == extractor_name,
+                EventListenerDB.access.owner == owner,
             )
         else:
+            extractor_name = extractor_info["name"]
+            extractor_db = EventListenerDB(
+                **extractor_info, properties=ExtractorInfo(**extractor_info)
+            )
+            logger.info(f"Received heartbeat from ${extractor_name}")
             existing_extractor = await EventListenerDB.find_one(
-                EventListenerDB.name == msg["queue"],
-                EventListenerDB.access.owner == owner,
+                EventListenerDB.name == extractor_name
             )
+
+        # check to see if extractor already exists and update if so
         if existing_extractor is not None:
             extractor_db.id = existing_extractor.id
             extractor_db.created = existing_extractor.created

From 792285aba357694e438c88f060105ea0b3fccc48 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Mon, 15 Apr 2024 09:41:06 -0500
Subject: [PATCH 08/30] Further clean up heartbeat listener

---
 backend/heartbeat_listener.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/backend/heartbeat_listener.py b/backend/heartbeat_listener.py
index de52b3601..728c8881f 100644
--- a/backend/heartbeat_listener.py
+++ b/backend/heartbeat_listener.py
@@ -34,14 +34,16 @@ async def callback(message: AbstractIncomingMessage):
         owner = msg["owner"]
         if owner is not None:
             # Extractor name should match queue, which includes secret key with common extractor_info["name"]
+            orig_properties = ExtractorInfo(**extractor_info)
             extractor_name = msg["queue"]
+            del extractor_info["name"]
             extractor_db = EventListenerDB(
                 **extractor_info,
-                extractor_name=extractor_name,
+                name=extractor_name,
                 access={"owner": owner},
-                properties=ExtractorInfo(**extractor_info),
+                properties=orig_properties,
             )
-            logger.info(f"Received heartbeat from ${extractor_name} owned by ${owner}")
+            logger.info(f"Received heartbeat from {extractor_name} owned by {owner}")
             existing_extractor = await EventListenerDB.find_one(
                 EventListenerDB.name == extractor_name,
                 EventListenerDB.access.owner == owner,
@@ -51,7 +53,7 @@ async def callback(message: AbstractIncomingMessage):
             extractor_db = EventListenerDB(
                 **extractor_info, properties=ExtractorInfo(**extractor_info)
             )
-            logger.info(f"Received heartbeat from ${extractor_name}")
+            logger.info(f"Received heartbeat from {extractor_name}")
             existing_extractor = await EventListenerDB.find_one(
                 EventListenerDB.name == extractor_name
             )

From 840ec34a989cff6e0a8d6ad4a04ffa5dd481a57a Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Mon, 15 Apr 2024 09:53:02 -0500
Subject: [PATCH 09/30] set alive status

---
 backend/heartbeat_listener.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/backend/heartbeat_listener.py b/backend/heartbeat_listener.py
index 728c8881f..39379adc7 100644
--- a/backend/heartbeat_listener.py
+++ b/backend/heartbeat_listener.py
@@ -72,9 +72,10 @@ async def callback(message: AbstractIncomingMessage):
                     % (extractor_name, existing_version, new_version)
                 )
 
+            # Update existing listeners alive status
             extractor_db.lastAlive = datetime.utcnow()
+            extractor_db.alive = True
             logger.info("%s is alive at %s" % (extractor_name, str(datetime.utcnow())))
-            # Update existing listeners alive status
             new_extractor = await extractor_db.replace()
             extractor_out = EventListenerOut(**new_extractor.dict())
 
@@ -83,6 +84,7 @@ async def callback(message: AbstractIncomingMessage):
         else:
             # Register new listener
             extractor_db.lastAlive = datetime.utcnow()
+            extractor_db.alive = True
             logger.info("%s is alive at %s" % (extractor_name, str(datetime.utcnow())))
             new_extractor = await extractor_db.insert()
             extractor_out = EventListenerOut(**new_extractor.dict())

From 7f539dd540d004e9c7ee8e07dfdd7f2827b1d944 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Mon, 15 Apr 2024 10:17:46 -0500
Subject: [PATCH 10/30] typo fix

---
 backend/app/routers/listeners.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index 279e114ee..42ae4921b 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -224,7 +224,7 @@ async def search_listeners(
     ]
     if not admin and not admin_mode:
         user_q = await GroupDB.find(
-            Or(GroupDB.creator == user_id, GroupDB.users.user.email == user),
+            Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id),
         ).to_list()
         user_groups = [u["_id"] for u in user_q]
 

From b5f9f6663090260a522c727422c8c549f7e57254 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Mon, 15 Apr 2024 10:55:46 -0500
Subject: [PATCH 11/30] Permission modification endpoints

---
 backend/app/routers/listeners.py              | 155 +++++++++++++++---
 .../listeners/ExtractionHistory.tsx           |   4 +-
 .../src/components/listeners/ListenerItem.tsx |   2 +-
 3 files changed, 135 insertions(+), 26 deletions(-)

diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index 42ae4921b..f9cfbd925 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -6,7 +6,6 @@
 
 from beanie import PydanticObjectId
 from beanie.operators import Or, RegEx, In, Exists
-from bson import ObjectId
 from fastapi import APIRouter, Depends, HTTPException
 from packaging import version
 
@@ -378,7 +377,7 @@ async def edit_listener(
         listener_id -- UUID of the listener to be udpated
         listener_in -- JSON object including updated information
     """
-    criteria_list = [EventListenerDB.id == ObjectId(listener_id)]
+    criteria_list = [EventListenerDB.id == PydanticObjectId(listener_id)]
     if not admin and not admin_mode:
         criteria_list.append(
             Or(
@@ -407,7 +406,7 @@ async def delete_listener(
     admin=Depends(get_admin),
 ):
     """Remove an Event Listener from the database. Will not clear event history for the listener."""
-    criteria_list = [EventListenerDB.id == ObjectId(listener_id)]
+    criteria_list = [EventListenerDB.id == PydanticObjectId(listener_id)]
     if not admin and not admin_mode:
         criteria_list.append(
             Or(
@@ -419,7 +418,7 @@ async def delete_listener(
     if listener:
         # unsubscribe the listener from any feeds
         async for feed in FeedDB.find(
-            FeedDB.listeners.listener_id == ObjectId(listener_id)
+            FeedDB.listeners.listener_id == PydanticObjectId(listener_id)
         ):
             await disassociate_listener_db(feed.id, listener_id)
         await listener.delete()
@@ -435,16 +434,18 @@ async def add_user_permission(
     admin_mode: bool = Depends(get_admin_mode),
     admin=Depends(get_admin),
 ):
-    criteria_list = [EventListenerDB.id == ObjectId(listener_id)]
-    if not admin and not admin_mode:
-        criteria_list.append(
-            Or(
-                Exists(EventListenerDB.access, False),
-                EventListenerDB.access.owner == user_id,
-            ),
-        )
-    listener = await EventListenerDB.find_one(*criteria_list)
-    if listener:
+    listener = await EventListenerDB.get(PydanticObjectId(listener_id))
+    if listener is not None:
+        if listener.access is None:
+            raise HTTPException(
+                status_code=403,
+                detail=f"listener {listener_id} does not require private access",
+            )
+        elif listener.access.owner != user_id and not (admin or admin_mode):
+            raise HTTPException(
+                status_code=403,
+                detail=f"please contact {listener.access.owner} to modify access",
+            )
         if target_user not in listener.access.users:
             listener.access.users.append(target_user)
             await listener.save()
@@ -460,16 +461,122 @@ async def remove_user_permission(
     admin_mode: bool = Depends(get_admin_mode),
     admin=Depends(get_admin),
 ):
-    criteria_list = [EventListenerDB.id == ObjectId(listener_id)]
-    if not admin and not admin_mode:
-        criteria_list.append(
-            Or(
-                Exists(EventListenerDB.access, False),
-                EventListenerDB.access.owner == user_id,
-            ),
-        )
-    listener = await EventListenerDB.find_one(*criteria_list)
-    if listener:
+    listener = await EventListenerDB.get(PydanticObjectId(listener_id))
+    if listener is not None:
+        if listener.access is None:
+            raise HTTPException(
+                status_code=403,
+                detail=f"listener {listener_id} does not require private access",
+            )
+        elif listener.access.owner != user_id and not (admin or admin_mode):
+            raise HTTPException(
+                status_code=403,
+                detail=f"please contact {listener.access.owner} to modify access",
+            )
         listener.access.users.remove(target_user)
         await listener.save()
     raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")
+
+
+@router.post("/{listener_id}/groups/{target_group}")
+async def add_group_permission(
+    listener_id: str,
+    target_group: str,
+    user_id=Depends(get_current_username),
+    admin_mode: bool = Depends(get_admin_mode),
+    admin=Depends(get_admin),
+):
+    listener = await EventListenerDB.get(PydanticObjectId(listener_id))
+    if listener is not None:
+        if listener.access is None:
+            raise HTTPException(
+                status_code=403,
+                detail=f"listener {listener_id} does not require private access",
+            )
+        elif listener.access.owner != user_id and not (admin or admin_mode):
+            raise HTTPException(
+                status_code=403,
+                detail=f"please contact {listener.access.owner} to modify access",
+            )
+        if PydanticObjectId(target_group) not in listener.access.groups:
+            listener.access.groups.append(PydanticObjectId(target_group))
+            await listener.save()
+        return listener.dict()
+    raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")
+
+
+@router.delete("/{listener_id}/groups/{target_group}")
+async def remove_group_permission(
+    listener_id: str,
+    target_group: str,
+    user_id=Depends(get_current_username),
+    admin_mode: bool = Depends(get_admin_mode),
+    admin=Depends(get_admin),
+):
+    listener = await EventListenerDB.get(PydanticObjectId(listener_id))
+    if listener is not None:
+        if listener.access is None:
+            raise HTTPException(
+                status_code=403,
+                detail=f"listener {listener_id} does not require private access",
+            )
+        elif listener.access.owner != user_id and not (admin or admin_mode):
+            raise HTTPException(
+                status_code=403,
+                detail=f"please contact {listener.access.owner} to modify access",
+            )
+        listener.access.users.remove(PydanticObjectId(target_group))
+        await listener.save()
+    raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")
+
+
+@router.post("/{listener_id}/datasets/{target_dataset}")
+async def add_dataset_permission(
+    listener_id: str,
+    target_dataset: str,
+    user_id=Depends(get_current_username),
+    admin_mode: bool = Depends(get_admin_mode),
+    admin=Depends(get_admin),
+):
+    listener = await EventListenerDB.get(PydanticObjectId(listener_id))
+    if listener is not None:
+        if listener.access is None:
+            raise HTTPException(
+                status_code=403,
+                detail=f"listener {listener_id} does not require private access",
+            )
+        elif listener.access.owner != user_id and not (admin or admin_mode):
+            raise HTTPException(
+                status_code=403,
+                detail=f"please contact {listener.access.owner} to modify access",
+            )
+        if PydanticObjectId(target_dataset) not in listener.access.datasets:
+            listener.access.datasets.append(PydanticObjectId(target_dataset))
+            await listener.save()
+        return listener.dict()
+    raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")
+
+
+@router.delete("/{listener_id}/users/{target_dataset}")
+async def remove_dataset_permission(
+    listener_id: str,
+    target_dataset: str,
+    user_id=Depends(get_current_username),
+    admin_mode: bool = Depends(get_admin_mode),
+    admin=Depends(get_admin),
+):
+    listener = await EventListenerDB.get(PydanticObjectId(listener_id))
+    if listener is not None:
+        if listener.access is None:
+            raise HTTPException(
+                status_code=403,
+                detail=f"listener {listener_id} does not require private access",
+            )
+        elif listener.access.owner != user_id and not (admin or admin_mode):
+            raise HTTPException(
+                status_code=403,
+                detail=f"please contact {listener.access.owner} to modify access",
+            )
+        listener.access.users.remove(PydanticObjectId(target_dataset))
+        await listener.save()
+    raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")
diff --git a/frontend/src/components/listeners/ExtractionHistory.tsx b/frontend/src/components/listeners/ExtractionHistory.tsx
index 47e4fe6c2..a63727e54 100644
--- a/frontend/src/components/listeners/ExtractionHistory.tsx
+++ b/frontend/src/components/listeners/ExtractionHistory.tsx
@@ -108,7 +108,9 @@ export const ExtractionHistory = (): JSX.Element => {
 												setSelectedExtractor(listener);
 											}}
 										>
-											<ListItemText primary={listener.name} />
+											<ListItemText
+												primary={listener.name.replace("private.", "")}
+											/>
 										</ListItemButton>
 									);
 								})
diff --git a/frontend/src/components/listeners/ListenerItem.tsx b/frontend/src/components/listeners/ListenerItem.tsx
index b149ded1a..da84f87a0 100644
--- a/frontend/src/components/listeners/ListenerItem.tsx
+++ b/frontend/src/components/listeners/ListenerItem.tsx
@@ -58,7 +58,7 @@ export default function ListenerItem(props: ListenerCardProps) {
 						setInfoOnly(true);
 					}}
 				>
-					{extractorName}
+					{extractorName.replace("private.", "")}
 				</Button>
 				{!(fileId !== undefined || datasetId !== undefined) ||
 				!extractor["alive"] ? (

From 5205b1078442fb27f63df91fa131a950d4736fe8 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Mon, 15 Apr 2024 11:06:01 -0500
Subject: [PATCH 12/30] Clean up permissions logic

---
 backend/app/routers/listeners.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index f9cfbd925..940218a0e 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -223,9 +223,9 @@ async def search_listeners(
     ]
     if not admin and not admin_mode:
         user_q = await GroupDB.find(
-            Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id),
+            Or(GroupDB.creator == user_id, GroupDB.users.email == user_id),
         ).to_list()
-        user_groups = [u["_id"] for u in user_q]
+        user_groups = [u.id for u in user_q]
 
         criteria_list.append(
             Or(
@@ -334,7 +334,7 @@ async def get_listeners(
         user_q = await GroupDB.find(
             Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id),
         ).to_list()
-        user_groups = [u["_id"] for u in user_q]
+        user_groups = [u.id for u in user_q]
 
         criteria_list.append(
             Or(
@@ -473,8 +473,10 @@ async def remove_user_permission(
                 status_code=403,
                 detail=f"please contact {listener.access.owner} to modify access",
             )
-        listener.access.users.remove(target_user)
-        await listener.save()
+        if target_user in listener.access.users:
+            listener.access.users.remove(target_user)
+            await listener.save()
+        return listener.dict()
     raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")
 
 
@@ -525,8 +527,10 @@ async def remove_group_permission(
                 status_code=403,
                 detail=f"please contact {listener.access.owner} to modify access",
             )
-        listener.access.users.remove(PydanticObjectId(target_group))
-        await listener.save()
+        if PydanticObjectId(target_group) in listener.access.groups:
+            listener.access.groups.remove(PydanticObjectId(target_group))
+            await listener.save()
+        return listener.dict()
     raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")
 
 
@@ -557,7 +561,7 @@ async def add_dataset_permission(
     raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")
 
 
-@router.delete("/{listener_id}/users/{target_dataset}")
+@router.delete("/{listener_id}/datasets/{target_dataset}")
 async def remove_dataset_permission(
     listener_id: str,
     target_dataset: str,
@@ -577,6 +581,8 @@ async def remove_dataset_permission(
                 status_code=403,
                 detail=f"please contact {listener.access.owner} to modify access",
             )
-        listener.access.users.remove(PydanticObjectId(target_dataset))
-        await listener.save()
+        if PydanticObjectId(target_dataset) in listener.access.datasets:
+            listener.access.datasets.remove(PydanticObjectId(target_dataset))
+            await listener.save()
+        return listener.dict()
     raise HTTPException(status_code=404, detail=f"listener {listener_id} not found")

From dcfb0c40c8d50bf876f77a65fdd3e2b3bbf4cdba Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Tue, 16 Apr 2024 08:15:36 -0500
Subject: [PATCH 13/30] run codegen

---
 frontend/src/openapi/v2/index.ts              |   1 +
 frontend/src/openapi/v2/models/AccessList.ts  |  14 ++
 .../src/openapi/v2/models/EventListenerIn.ts  |   3 +
 .../src/openapi/v2/models/EventListenerOut.ts |   2 +
 .../src/openapi/v2/models/ExtractorInfo.ts    |   1 +
 .../v2/models/LegacyEventListenerIn.ts        |   1 +
 .../openapi/v2/services/ListenersService.ts   | 166 ++++++++++++++++++
 7 files changed, 188 insertions(+)
 create mode 100644 frontend/src/openapi/v2/models/AccessList.ts

diff --git a/frontend/src/openapi/v2/index.ts b/frontend/src/openapi/v2/index.ts
index 0aeaafe35..66fec22f0 100644
--- a/frontend/src/openapi/v2/index.ts
+++ b/frontend/src/openapi/v2/index.ts
@@ -5,6 +5,7 @@ export { ApiError } from './core/ApiError';
 export { CancelablePromise } from './core/CancelablePromise';
 export { OpenAPI } from './core/OpenAPI';
 
+export type { AccessList } from './models/AccessList';
 export type { AuthorizationBase } from './models/AuthorizationBase';
 export type { AuthorizationMetadata } from './models/AuthorizationMetadata';
 export type { AuthorizationOut } from './models/AuthorizationOut';
diff --git a/frontend/src/openapi/v2/models/AccessList.ts b/frontend/src/openapi/v2/models/AccessList.ts
new file mode 100644
index 000000000..f366c3d85
--- /dev/null
+++ b/frontend/src/openapi/v2/models/AccessList.ts
@@ -0,0 +1,14 @@
+/* istanbul ignore file */
+/* tslint:disable */
+/* eslint-disable */
+
+/**
+ * Container object for lists of user emails/group IDs/dataset IDs that can submit to listener.
+ * The singular owner is the primary who can modify other lists.
+ */
+export type AccessList = {
+    owner: string;
+    users?: Array<string>;
+    groups?: Array<string>;
+    datasets?: Array<string>;
+}
diff --git a/frontend/src/openapi/v2/models/EventListenerIn.ts b/frontend/src/openapi/v2/models/EventListenerIn.ts
index dc04fb350..ac1397f72 100644
--- a/frontend/src/openapi/v2/models/EventListenerIn.ts
+++ b/frontend/src/openapi/v2/models/EventListenerIn.ts
@@ -2,6 +2,8 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { AccessList } from './AccessList';
+
 /**
  * On submission, minimum info for a listener is name, version and description. Clowder will use name and version to locate queue.
  */
@@ -9,4 +11,5 @@ export type EventListenerIn = {
     name: string;
     version?: string;
     description?: string;
+    access?: AccessList;
 }
diff --git a/frontend/src/openapi/v2/models/EventListenerOut.ts b/frontend/src/openapi/v2/models/EventListenerOut.ts
index a6a57f20f..2c990c35b 100644
--- a/frontend/src/openapi/v2/models/EventListenerOut.ts
+++ b/frontend/src/openapi/v2/models/EventListenerOut.ts
@@ -2,6 +2,7 @@
 /* tslint:disable */
 /* eslint-disable */
 
+import type { AccessList } from './AccessList';
 import type { ExtractorInfo } from './ExtractorInfo';
 import type { UserOut } from './UserOut';
 
@@ -12,6 +13,7 @@ export type EventListenerOut = {
     name: string;
     version?: string;
     description?: string;
+    access?: AccessList;
     id?: string;
     creator?: UserOut;
     created?: string;
diff --git a/frontend/src/openapi/v2/models/ExtractorInfo.ts b/frontend/src/openapi/v2/models/ExtractorInfo.ts
index 292a54094..6c2fee09b 100644
--- a/frontend/src/openapi/v2/models/ExtractorInfo.ts
+++ b/frontend/src/openapi/v2/models/ExtractorInfo.ts
@@ -22,4 +22,5 @@ export type ExtractorInfo = {
     categories?: Array<string>;
     parameters?: any;
     version?: string;
+    unique_key?: string;
 }
diff --git a/frontend/src/openapi/v2/models/LegacyEventListenerIn.ts b/frontend/src/openapi/v2/models/LegacyEventListenerIn.ts
index 6b634351f..bc5e504b3 100644
--- a/frontend/src/openapi/v2/models/LegacyEventListenerIn.ts
+++ b/frontend/src/openapi/v2/models/LegacyEventListenerIn.ts
@@ -22,5 +22,6 @@ export type LegacyEventListenerIn = {
     categories?: Array<string>;
     parameters?: any;
     version?: string;
+    unique_key?: string;
     description?: string;
 }
diff --git a/frontend/src/openapi/v2/services/ListenersService.ts b/frontend/src/openapi/v2/services/ListenersService.ts
index d0fa80add..fefe5c17b 100644
--- a/frontend/src/openapi/v2/services/ListenersService.ts
+++ b/frontend/src/openapi/v2/services/ListenersService.ts
@@ -39,6 +39,7 @@ export class ListenersService {
      * @param label
      * @param aliveOnly
      * @param process
+     * @param datasetId
      * @returns Paged Successful Response
      * @throws ApiError
      */
@@ -50,6 +51,7 @@ export class ListenersService {
         label?: string,
         aliveOnly: boolean = false,
         process?: string,
+        datasetId?: string,
     ): CancelablePromise<Paged> {
         return __request({
             method: 'GET',
@@ -62,6 +64,7 @@ export class ListenersService {
                 'label': label,
                 'alive_only': aliveOnly,
                 'process': process,
+                'dataset_id': datasetId,
             },
             errors: {
                 422: `Validation Error`,
@@ -103,6 +106,7 @@ export class ListenersService {
      * @param limit
      * @param heartbeatInterval
      * @param process
+     * @param datasetId
      * @returns Paged Successful Response
      * @throws ApiError
      */
@@ -112,6 +116,7 @@ export class ListenersService {
         limit: number = 2,
         heartbeatInterval: number = 300,
         process?: string,
+        datasetId?: string,
     ): CancelablePromise<Paged> {
         return __request({
             method: 'GET',
@@ -122,6 +127,7 @@ export class ListenersService {
                 'limit': limit,
                 'heartbeat_interval': heartbeatInterval,
                 'process': process,
+                'dataset_id': datasetId,
             },
             errors: {
                 422: `Validation Error`,
@@ -183,16 +189,21 @@ export class ListenersService {
      * listener_in -- JSON object including updated information
      * @param listenerId
      * @param requestBody
+     * @param datasetId
      * @returns EventListenerOut Successful Response
      * @throws ApiError
      */
     public static editListenerApiV2ListenersListenerIdPut(
         listenerId: string,
         requestBody: EventListenerIn,
+        datasetId?: string,
     ): CancelablePromise<EventListenerOut> {
         return __request({
             method: 'PUT',
             path: `/api/v2/listeners/${listenerId}`,
+            query: {
+                'dataset_id': datasetId,
+            },
             body: requestBody,
             mediaType: 'application/json',
             errors: {
@@ -205,15 +216,20 @@ export class ListenersService {
      * Delete Listener
      * Remove an Event Listener from the database. Will not clear event history for the listener.
      * @param listenerId
+     * @param datasetId
      * @returns any Successful Response
      * @throws ApiError
      */
     public static deleteListenerApiV2ListenersListenerIdDelete(
         listenerId: string,
+        datasetId?: string,
     ): CancelablePromise<any> {
         return __request({
             method: 'DELETE',
             path: `/api/v2/listeners/${listenerId}`,
+            query: {
+                'dataset_id': datasetId,
+            },
             errors: {
                 422: `Validation Error`,
             },
@@ -244,4 +260,154 @@ export class ListenersService {
         });
     }
 
+    /**
+     * Add User Permission
+     * @param listenerId
+     * @param targetUser
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static addUserPermissionApiV2ListenersListenerIdUsersTargetUserPost(
+        listenerId: string,
+        targetUser: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners/${listenerId}/users/${targetUser}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
+
+    /**
+     * Remove User Permission
+     * @param listenerId
+     * @param targetUser
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static removeUserPermissionApiV2ListenersListenerIdUsersTargetUserDelete(
+        listenerId: string,
+        targetUser: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}/users/${targetUser}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
+
+    /**
+     * Add Group Permission
+     * @param listenerId
+     * @param targetGroup
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static addGroupPermissionApiV2ListenersListenerIdGroupsTargetGroupPost(
+        listenerId: string,
+        targetGroup: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners/${listenerId}/groups/${targetGroup}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
+
+    /**
+     * Remove Group Permission
+     * @param listenerId
+     * @param targetGroup
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static removeGroupPermissionApiV2ListenersListenerIdGroupsTargetGroupDelete(
+        listenerId: string,
+        targetGroup: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}/groups/${targetGroup}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
+
+    /**
+     * Add Dataset Permission
+     * @param listenerId
+     * @param targetDataset
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static addDatasetPermissionApiV2ListenersListenerIdDatasetsTargetDatasetPost(
+        listenerId: string,
+        targetDataset: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners/${listenerId}/datasets/${targetDataset}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
+
+    /**
+     * Remove Dataset Permission
+     * @param listenerId
+     * @param targetDataset
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static removeDatasetPermissionApiV2ListenersListenerIdDatasetsTargetDatasetDelete(
+        listenerId: string,
+        targetDataset: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}/datasets/${targetDataset}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
+
 }
\ No newline at end of file

From 65ba16aae88c7ffc01705427e56a4e8c2c2a3413 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Tue, 16 Apr 2024 08:27:14 -0500
Subject: [PATCH 14/30] Fix process logic if no process rules given

---
 backend/app/routers/listeners.py | 36 ++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index d34d28e25..63c3c5a02 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -219,11 +219,25 @@ async def search_listeners(
     if process:
         if process == "file":
             aggregation_pipeline.append(
-                {"$match": {"properties.process.file": {"$exists": True}}}
+                {
+                    "$match": {
+                        "$or": [
+                            {"properties.process.file": {"$exists": True}},
+                            {"properties.process.file": {}},
+                        ]
+                    }
+                }
             )
         if process == "dataset":
             aggregation_pipeline.append(
-                {"$match": {"properties.process.dataset": {"$exists": True}}}
+                {
+                    "$match": {
+                        "$or": [
+                            {"properties.process.dataset": {"$exists": True}},
+                            {"properties.process.dataset": {}},
+                        ]
+                    }
+                }
             )
     # Add pagination
     aggregation_pipeline.append(
@@ -341,11 +355,25 @@ async def get_listeners(
     if process:
         if process == "file":
             aggregation_pipeline.append(
-                {"$match": {"properties.process.file": {"$exists": True}}}
+                {
+                    "$match": {
+                        "$or": [
+                            {"properties.process.file": {"$exists": True}},
+                            {"properties.process": {}},
+                        ]
+                    }
+                }
             )
         if process == "dataset":
             aggregation_pipeline.append(
-                {"$match": {"properties.process.dataset": {"$exists": True}}}
+                {
+                    "$match": {
+                        "$or": [
+                            {"properties.process.dataset": {"$exists": True}},
+                            {"properties.process": {}},
+                        ]
+                    }
+                }
             )
     # Add pagination
     aggregation_pipeline.append(

From c88bca5faf5e7cb01280950b03eb5d3fb4bb5a20 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Tue, 16 Apr 2024 08:55:57 -0500
Subject: [PATCH 15/30] enforce permissions in more places

---
 backend/app/routers/feeds.py     | 72 +++++++++++++++++++++++++++++---
 backend/app/routers/listeners.py |  2 +
 2 files changed, 68 insertions(+), 6 deletions(-)

diff --git a/backend/app/routers/feeds.py b/backend/app/routers/feeds.py
index 42f9dc110..fdc923743 100644
--- a/backend/app/routers/feeds.py
+++ b/backend/app/routers/feeds.py
@@ -1,15 +1,18 @@
 from typing import List, Optional
 
+from beanie import PydanticObjectId
+from beanie.operators import Or
+from fastapi import APIRouter, Depends, HTTPException
+from pika.adapters.blocking_connection import BlockingChannel
+
 from app.keycloak_auth import get_current_user, get_current_username
 from app.models.feeds import FeedDB, FeedIn, FeedOut
 from app.models.files import FileOut
+from app.models.groups import GroupDB
 from app.models.listeners import EventListenerDB, FeedListener
 from app.models.users import UserOut
 from app.rabbitmq.listeners import submit_file_job
 from app.search.connect import check_search_result
-from beanie import PydanticObjectId
-from fastapi import APIRouter, Depends, HTTPException
-from pika.adapters.blocking_connection import BlockingChannel
 
 router = APIRouter()
 
@@ -48,6 +51,26 @@ async def check_feed_listeners(
         if (
             listener_info := await EventListenerDB.get(PydanticObjectId(targ_listener))
         ) is not None:
+            if (
+                listener_info.access is not None
+                and not user.admin
+                and not user.admin_mode
+            ):
+                dataset_id = file_out.dataset_id
+                user_id = user.email
+                group_q = await GroupDB.find(
+                    Or(GroupDB.creator == user_id, GroupDB.users.email == user_id),
+                ).to_list()
+                user_groups = [g.id for g in group_q]
+
+                valid_submission = (
+                    (listener_info.access.owner == user_id)
+                    or (user.email in listener_info.access.users)
+                    or (dataset_id in listener_info.access.datasets)
+                    or (not set(user_groups).isdisjoint(listener_info.access.groups))
+                )
+                if not valid_submission:
+                    continue
             await submit_file_job(
                 file_out,
                 listener_info.name,  # routing_key
@@ -121,7 +144,7 @@ async def delete_feed(
 async def associate_listener(
     feed_id: str,
     listener: FeedListener,
-    user=Depends(get_current_user),
+    user_id=Depends(get_current_username()),
 ):
     """Associate an existing Event Listener with a Feed, e.g. so it will be triggered on new Feed results.
 
@@ -131,8 +154,26 @@ async def associate_listener(
     """
     if (feed := await FeedDB.get(PydanticObjectId(feed_id))) is not None:
         if (
-            await EventListenerDB.get(PydanticObjectId(listener.listener_id))
+            listener_obj := await EventListenerDB.get(
+                PydanticObjectId(listener.listener_id)
+            )
         ) is not None:
+            if listener_obj.access is not None:
+                group_q = await GroupDB.find(
+                    Or(GroupDB.creator == user_id, GroupDB.users.email == user_id),
+                ).to_list()
+                user_groups = [g.id for g in group_q]
+
+                valid_modificaiton = (
+                    (listener_obj.access.owner == user_id)
+                    or (user_id in listener_obj.access.users)
+                    or (not set(user_groups).isdisjoint(listener_obj.access.groups))
+                )
+                if not valid_modificaiton:
+                    raise HTTPException(
+                        status_code=403,
+                        detail=f"Insufficient permissions for this listener",
+                    )
             feed.listeners.append(listener)
             await feed.save()
             return feed.dict()
@@ -146,7 +187,7 @@ async def associate_listener(
 async def disassociate_listener(
     feed_id: str,
     listener_id: str,
-    user=Depends(get_current_user),
+    user_id=Depends(get_current_username),
 ):
     """Disassociate an Event Listener from a Feed.
 
@@ -155,6 +196,25 @@ async def disassociate_listener(
         listener_id: UUID of Event Listener that should be disassociated
     """
     if (await FeedDB.get(PydanticObjectId(feed_id))) is not None:
+        if (
+            listener_obj := await EventListenerDB.get(PydanticObjectId(listener_id))
+        ) is not None:
+            if listener_obj.access is not None:
+                group_q = await GroupDB.find(
+                    Or(GroupDB.creator == user_id, GroupDB.users.email == user_id),
+                ).to_list()
+                user_groups = [g.id for g in group_q]
+
+                valid_modificaiton = (
+                    (listener_obj.access.owner == user_id)
+                    or (user_id in listener_obj.access.users)
+                    or (not set(user_groups).isdisjoint(listener_obj.access.groups))
+                )
+                if not valid_modificaiton:
+                    raise HTTPException(
+                        status_code=403,
+                        detail=f"Insufficient permissions for this listener",
+                    )
         await disassociate_listener_db(feed_id, listener_id)
         return {"disassociated": listener_id}
     raise HTTPException(status_code=404, detail=f"feed {feed_id} not found")
diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index 63c3c5a02..1b3d8b754 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -431,6 +431,7 @@ async def edit_listener(
     """
     criteria_list = [EventListenerDB.id == PydanticObjectId(listener_id)]
     if not admin and not admin_mode:
+        # Must either be owner, or a listener with no restrictions
         criteria_list.append(
             Or(
                 Exists(EventListenerDB.access, False),
@@ -460,6 +461,7 @@ async def delete_listener(
     """Remove an Event Listener from the database. Will not clear event history for the listener."""
     criteria_list = [EventListenerDB.id == PydanticObjectId(listener_id)]
     if not admin and not admin_mode:
+        # Must either be owner, or a listener with no restrictions
         criteria_list.append(
             Or(
                 Exists(EventListenerDB.access, False),

From da06adc87ea46e284b52da3f3c90556cecbe76bc Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Tue, 16 Apr 2024 09:02:55 -0500
Subject: [PATCH 16/30] fix pytest

---
 backend/app/routers/feeds.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/app/routers/feeds.py b/backend/app/routers/feeds.py
index fdc923743..f34bc9883 100644
--- a/backend/app/routers/feeds.py
+++ b/backend/app/routers/feeds.py
@@ -144,7 +144,7 @@ async def delete_feed(
 async def associate_listener(
     feed_id: str,
     listener: FeedListener,
-    user_id=Depends(get_current_username()),
+    user_id=Depends(get_current_username),
 ):
     """Associate an existing Event Listener with a Feed, e.g. so it will be triggered on new Feed results.
 

From 4db0a06f647e0d188e035ab168363ecd1071af2f Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Tue, 16 Apr 2024 09:43:09 -0500
Subject: [PATCH 17/30] clean up dataset permission logic

---
 backend/app/routers/listeners.py              | 54 +++++++++++++-----
 frontend/src/actions/listeners.js             | 20 +++++--
 .../src/components/listeners/Listeners.tsx    | 56 ++++++++++++++++---
 .../openapi/v2/services/ListenersService.ts   |  2 +
 4 files changed, 105 insertions(+), 27 deletions(-)

diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index 1b3d8b754..385cc1f37 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -200,6 +200,7 @@ async def search_listeners(
     heartbeat_interval: Optional[int] = settings.listener_heartbeat_interval,
     user_id=Depends(get_current_username),
     process: Optional[str] = None,
+    dataset_id: Optional[str] = None,
     admin_mode: bool = Depends(get_admin_mode),
     admin=Depends(get_admin),
 ):
@@ -256,14 +257,25 @@ async def search_listeners(
         ).to_list()
         user_groups = [u.id for u in user_q]
 
-        criteria_list.append(
-            Or(
-                Exists(EventListenerDB.access, False),
-                EventListenerDB.access.owner == user_id,
-                EventListenerDB.access.users == user_id,
-                In(EventListenerDB.access.groups, user_groups),
+        if dataset_id is None:
+            criteria_list.append(
+                Or(
+                    Exists(EventListenerDB.access, False),
+                    EventListenerDB.access.owner == user_id,
+                    EventListenerDB.access.users == user_id,
+                    In(EventListenerDB.access.groups, user_groups),
+                )
+            )
+        else:
+            criteria_list.append(
+                Or(
+                    Exists(EventListenerDB.access, False),
+                    EventListenerDB.access.owner == user_id,
+                    EventListenerDB.access.users == user_id,
+                    In(EventListenerDB.access.groups, user_groups),
+                    EventListenerDB.access.datasets == PydanticObjectId(dataset_id),
+                )
             )
-        )
 
     listeners_and_count = (
         await EventListenerDB.find(*criteria_list)
@@ -327,6 +339,7 @@ async def get_listeners(
     label: Optional[str] = None,
     alive_only: Optional[bool] = False,
     process: Optional[str] = None,
+    dataset_id: Optional[str] = None,
     admin_mode: bool = Depends(get_admin_mode),
     admin=Depends(get_admin),
 ):
@@ -339,6 +352,8 @@ async def get_listeners(
         category -- filter by category has to be exact match
         label -- filter by label has to be exact match
         alive_only -- filter by alive status
+        process -- filter by file or dataset type (if specified)
+        dataset_id -- restrict to listeners that run on the given dataset or a file within (if not otherwise permitted)
     """
     # First compute alive flag for all listeners
     aggregation_pipeline = [
@@ -388,14 +403,25 @@ async def get_listeners(
         ).to_list()
         user_groups = [u.id for u in user_q]
 
-        criteria_list.append(
-            Or(
-                Exists(EventListenerDB.access, False),
-                EventListenerDB.access.owner == user_id,
-                EventListenerDB.access.users == user_id,
-                In(EventListenerDB.access.groups, user_groups),
+        if dataset_id is None:
+            criteria_list.append(
+                Or(
+                    Exists(EventListenerDB.access, False),
+                    EventListenerDB.access.owner == user_id,
+                    EventListenerDB.access.users == user_id,
+                    In(EventListenerDB.access.groups, user_groups),
+                )
+            )
+        else:
+            criteria_list.append(
+                Or(
+                    Exists(EventListenerDB.access, False),
+                    EventListenerDB.access.owner == user_id,
+                    EventListenerDB.access.users == user_id,
+                    In(EventListenerDB.access.groups, user_groups),
+                    EventListenerDB.access.datasets == PydanticObjectId(dataset_id),
+                )
             )
-        )
 
     # Run aggregate query and return
     # Sort by name alphabetically
diff --git a/frontend/src/actions/listeners.js b/frontend/src/actions/listeners.js
index 3c6b8633e..814030f4e 100644
--- a/frontend/src/actions/listeners.js
+++ b/frontend/src/actions/listeners.js
@@ -11,6 +11,7 @@ export function fetchListeners(
 	label = null,
 	aliveOnly = false,
 	process = null,
+	dataset_id = null
 ) {
 	return (dispatch) => {
 		// TODO: Parameters for dates? paging?
@@ -21,7 +22,8 @@ export function fetchListeners(
 			category,
 			label,
 			aliveOnly,
-			process
+			process,
+			dataset_id
 		)
 			.then((json) => {
 				dispatch({
@@ -41,7 +43,8 @@ export function fetchListeners(
 							category,
 							label,
 							aliveOnly,
-							process
+							process,
+							dataset_id
 						)
 					)
 				);
@@ -57,6 +60,7 @@ export function queryListeners(
 	limit = 21,
 	heartbeatInterval = 0,
 	process = null,
+	datasetId = null
 ) {
 	return (dispatch) => {
 		// TODO: Parameters for dates? paging?
@@ -65,7 +69,8 @@ export function queryListeners(
 			skip,
 			limit,
 			heartbeatInterval,
-			process
+			process,
+			datasetId
 		)
 			.then((json) => {
 				dispatch({
@@ -78,7 +83,14 @@ export function queryListeners(
 				dispatch(
 					handleErrors(
 						reason,
-						queryListeners(text, skip, limit, heartbeatInterval, process)
+						queryListeners(
+							text,
+							skip,
+							limit,
+							heartbeatInterval,
+							process,
+							datasetId
+						)
 					)
 				);
 			});
diff --git a/frontend/src/components/listeners/Listeners.tsx b/frontend/src/components/listeners/Listeners.tsx
index c797834c6..bacada056 100644
--- a/frontend/src/components/listeners/Listeners.tsx
+++ b/frontend/src/components/listeners/Listeners.tsx
@@ -45,7 +45,7 @@ export function Listeners(props: ListenerProps) {
 		selectedCategory: string | null,
 		selectedLabel: string | null,
 		aliveOnly: boolean | undefined,
-		process: string | undefined,
+		process: string | undefined
 	) =>
 		dispatch(
 			fetchListeners(
@@ -55,7 +55,8 @@ export function Listeners(props: ListenerProps) {
 				selectedCategory,
 				selectedLabel,
 				aliveOnly,
-				process
+				process,
+				datasetId
 			)
 		);
 	const searchListeners = (
@@ -63,8 +64,11 @@ export function Listeners(props: ListenerProps) {
 		skip: number | undefined,
 		limit: number | undefined,
 		heartbeatInterval: number | undefined,
-		process: string | undefined,
-	) => dispatch(queryListeners(text, skip, limit, heartbeatInterval, process));
+		process: string | undefined
+	) =>
+		dispatch(
+			queryListeners(text, skip, limit, heartbeatInterval, process, datasetId)
+		);
 	const listAvailableCategories = () => dispatch(fetchListenerCategories());
 	const listAvailableLabels = () => dispatch(fetchListenerLabels());
 
@@ -104,14 +108,31 @@ export function Listeners(props: ListenerProps) {
 		setSelectedCategory("");
 
 		if (searchText !== "") searchListeners(searchText, 0, limit, 0, process);
-		else listListeners(0, limit, 0, selectedCategory, selectedLabel, aliveOnly, process);
+		else
+			listListeners(
+				0,
+				limit,
+				0,
+				selectedCategory,
+				selectedLabel,
+				aliveOnly,
+				process
+			);
 	}, [searchText]);
 
 	useEffect(() => {
 		// reset page and reset search text with each new search term
 		setCurrPageNum(1);
 		setSearchText("");
-		listListeners(0, limit, 0, selectedCategory, selectedLabel, aliveOnly, process);
+		listListeners(
+			0,
+			limit,
+			0,
+			selectedCategory,
+			selectedLabel,
+			aliveOnly,
+			process
+		);
 	}, [aliveOnly]);
 
 	// any of the change triggers timer to fetch the extractor status
@@ -154,20 +175,37 @@ export function Listeners(props: ListenerProps) {
 		const selectedCategoryValue = (event.target as HTMLInputElement).value;
 		setSelectedCategory(selectedCategoryValue);
 		setSearchText("");
-		listListeners(0, limit, 0, selectedCategoryValue, selectedLabel, aliveOnly, process);
+		listListeners(
+			0,
+			limit,
+			0,
+			selectedCategoryValue,
+			selectedLabel,
+			aliveOnly,
+			process
+		);
 	};
 
 	const handleLabelChange = (event: React.ChangeEvent<HTMLInputElement>) => {
 		const selectedLabelValue = (event.target as HTMLInputElement).value;
 		setSelectedLabel(selectedLabelValue);
 		setSearchText("");
-		listListeners(0, limit, 0, selectedCategory, selectedLabelValue, aliveOnly, process);
+		listListeners(
+			0,
+			limit,
+			0,
+			selectedCategory,
+			selectedLabelValue,
+			aliveOnly,
+			process
+		);
 	};
 
 	const handlePageChange = (_: ChangeEvent<unknown>, value: number) => {
 		const newSkip = (value - 1) * limit;
 		setCurrPageNum(value);
-		if (searchText !== "") searchListeners(searchText, newSkip, limit, 0, process);
+		if (searchText !== "")
+			searchListeners(searchText, newSkip, limit, 0, process);
 		else listListeners(newSkip, limit, 0, null, null, aliveOnly, process);
 	};
 
diff --git a/frontend/src/openapi/v2/services/ListenersService.ts b/frontend/src/openapi/v2/services/ListenersService.ts
index fefe5c17b..e0a744d92 100644
--- a/frontend/src/openapi/v2/services/ListenersService.ts
+++ b/frontend/src/openapi/v2/services/ListenersService.ts
@@ -32,6 +32,8 @@ export class ListenersService {
      * category -- filter by category has to be exact match
      * label -- filter by label has to be exact match
      * alive_only -- filter by alive status
+     * process -- filter by file or dataset type (if specified)
+     * dataset_id -- restrict to listeners that run on the given dataset or a file within (if not otherwise permitted)
      * @param skip
      * @param limit
      * @param heartbeatInterval

From baaccf2d7e769f03028feb5f480aaac406a32468 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Thu, 25 Apr 2024 09:01:35 -0500
Subject: [PATCH 18/30] revert admin_mode logic checks

---
 backend/app/routers/authorization.py | 2 +-
 backend/app/routers/groups.py        | 4 ++--
 backend/app/routers/listeners.py     | 8 ++++----
 3 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/backend/app/routers/authorization.py b/backend/app/routers/authorization.py
index 10b1b3bcc..d4ef9e3a7 100644
--- a/backend/app/routers/authorization.py
+++ b/backend/app/routers/authorization.py
@@ -74,7 +74,7 @@ async def get_dataset_role(
     """Retrieve role of user for a specific dataset."""
     # Get group id and the associated users from authorization
     criteria = []
-    if not admin and not admin_mode:
+    if not admin or not admin_mode:
         criteria.append(
             Or(
                 AuthorizationDB.creator == current_user,
diff --git a/backend/app/routers/groups.py b/backend/app/routers/groups.py
index f807f194e..1434d806a 100644
--- a/backend/app/routers/groups.py
+++ b/backend/app/routers/groups.py
@@ -46,7 +46,7 @@ async def get_groups(
 
     """
     criteria_list = []
-    if not admin and not admin_mode:
+    if not admin or not admin_mode:
         criteria_list.append(
             Or(
                 GroupDB.creator == user_id,
@@ -96,7 +96,7 @@ async def search_group(
             RegEx(field=GroupDB.description, pattern=search_term),
         ),
     ]
-    if not admin and not admin_mode:
+    if not admin or not admin_mode:
         criteria_list.append(
             Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id)
         )
diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index 385cc1f37..ecd4e83ee 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -251,7 +251,7 @@ async def search_listeners(
             RegEx(field=EventListenerDB.description, pattern=text),
         )
     ]
-    if not admin and not admin_mode:
+    if not admin or not admin_mode:
         user_q = await GroupDB.find(
             Or(GroupDB.creator == user_id, GroupDB.users.email == user_id),
         ).to_list()
@@ -397,7 +397,7 @@ async def get_listeners(
 
     # Filter by ownership
     criteria_list = []
-    if not admin and not admin_mode:
+    if not admin or not admin_mode:
         user_q = await GroupDB.find(
             Or(GroupDB.creator == user_id, GroupDB.users.user.email == user_id),
         ).to_list()
@@ -456,7 +456,7 @@ async def edit_listener(
         listener_in -- JSON object including updated information
     """
     criteria_list = [EventListenerDB.id == PydanticObjectId(listener_id)]
-    if not admin and not admin_mode:
+    if not admin or not admin_mode:
         # Must either be owner, or a listener with no restrictions
         criteria_list.append(
             Or(
@@ -486,7 +486,7 @@ async def delete_listener(
 ):
     """Remove an Event Listener from the database. Will not clear event history for the listener."""
     criteria_list = [EventListenerDB.id == PydanticObjectId(listener_id)]
-    if not admin and not admin_mode:
+    if not admin or not admin_mode:
         # Must either be owner, or a listener with no restrictions
         criteria_list.append(
             Or(

From 9cd5f964502825defb5f3f26468906da8976e026 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Thu, 25 Apr 2024 09:46:47 -0500
Subject: [PATCH 19/30] clean up test logic

---
 backend/app/routers/listeners.py | 14 +++++++-------
 backend/app/tests/utils.py       |  6 +++---
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index ecd4e83ee..23b179d4c 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -5,7 +5,7 @@
 from typing import List, Optional
 
 from beanie import PydanticObjectId
-from beanie.operators import Or, RegEx, In, Exists
+from beanie.operators import Or, RegEx, In, NE
 from fastapi import APIRouter, Depends, HTTPException
 from packaging import version
 
@@ -260,7 +260,7 @@ async def search_listeners(
         if dataset_id is None:
             criteria_list.append(
                 Or(
-                    Exists(EventListenerDB.access, False),
+                    EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
                     EventListenerDB.access.users == user_id,
                     In(EventListenerDB.access.groups, user_groups),
@@ -269,7 +269,7 @@ async def search_listeners(
         else:
             criteria_list.append(
                 Or(
-                    Exists(EventListenerDB.access, False),
+                    EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
                     EventListenerDB.access.users == user_id,
                     In(EventListenerDB.access.groups, user_groups),
@@ -406,7 +406,7 @@ async def get_listeners(
         if dataset_id is None:
             criteria_list.append(
                 Or(
-                    Exists(EventListenerDB.access, False),
+                    EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
                     EventListenerDB.access.users == user_id,
                     In(EventListenerDB.access.groups, user_groups),
@@ -415,7 +415,7 @@ async def get_listeners(
         else:
             criteria_list.append(
                 Or(
-                    Exists(EventListenerDB.access, False),
+                    EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
                     EventListenerDB.access.users == user_id,
                     In(EventListenerDB.access.groups, user_groups),
@@ -460,7 +460,7 @@ async def edit_listener(
         # Must either be owner, or a listener with no restrictions
         criteria_list.append(
             Or(
-                Exists(EventListenerDB.access, False),
+                EventListenerDB.access == None,
                 EventListenerDB.access.owner == user_id,
             ),
         )
@@ -490,7 +490,7 @@ async def delete_listener(
         # Must either be owner, or a listener with no restrictions
         criteria_list.append(
             Or(
-                Exists(EventListenerDB.access, False),
+                EventListenerDB.access == None,
                 EventListenerDB.access.owner == user_id,
             ),
         )
diff --git a/backend/app/tests/utils.py b/backend/app/tests/utils.py
index 19c2ee380..552f1fbbe 100644
--- a/backend/app/tests/utils.py
+++ b/backend/app/tests/utils.py
@@ -253,7 +253,7 @@ def register_v1_extractor(
     client: TestClient, headers: dict, name: str = None, user: str = None
 ):
     """Registers a new v1 listener (extractor) and returns the JSON."""
-    new_extractor = extractor_info_v1_example
+    new_extractor = dict(extractor_info_v1_example)
     if name:
         new_extractor["name"] = name
     if user:
@@ -268,11 +268,11 @@ def register_v1_extractor(
 
 def register_v2_listener(client: TestClient, headers: dict, name: str = None):
     """Registers a new v2 listener and returns the JSON. Note that this typically uses RabbitMQ heartbeat."""
-    listener_info = listener_v2_example
+    listener_info = dict(listener_v2_example)
     if name is not None:
         listener_info["name"] = name
     response = client.post(
-        f"{settings.API_V2_STR}/listeners", json=listener_v2_example, headers=headers
+        f"{settings.API_V2_STR}/listeners", json=listener_info, headers=headers
     )
     assert response.status_code == 200
     assert response.json().get("id") is not None

From b1a8ae3da0201dfbedf03362a7482a95c58c4b4e Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Wed, 22 May 2024 08:39:07 -0500
Subject: [PATCH 20/30] Update authorization_deps.py

---
 backend/app/deps/authorization_deps.py | 31 +++++++++++++++++++++-----
 1 file changed, 25 insertions(+), 6 deletions(-)

diff --git a/backend/app/deps/authorization_deps.py b/backend/app/deps/authorization_deps.py
index 308406e4f..eafabe4a5 100644
--- a/backend/app/deps/authorization_deps.py
+++ b/backend/app/deps/authorization_deps.py
@@ -395,13 +395,13 @@ class ListenerAuthorization:
     For more info see https://fastapi.tiangolo.com/advanced/advanced-dependencies/.
     Regular users are not allowed to run non-active listeners"""
 
-    # def __init__(self, optional_arg: str = None):
-    #         self.optional_arg = optional_arg
+    # def __init__(self, role: str = "viewer"):
+    #     self.role = role
 
     async def __call__(
         self,
         listener_id: str,
-        current_user: str = Depends(get_current_username),
+        username: str = Depends(get_current_username),
         admin_mode: bool = Depends(get_admin_mode),
         admin: bool = Depends(get_admin),
     ):
@@ -409,18 +409,37 @@ async def __call__(
         if admin and admin_mode:
             return True
 
-        # Else check if listener is active or current user is the creator of the extractor
         if (
             listener := await EventListenerDB.get(PydanticObjectId(listener_id))
         ) is not None:
+            # If listener has access restrictions, evaluate them against requesting user
+            if listener.access is not None:
+                group_q = await GroupDB.find(
+                    Or(GroupDB.creator == username, GroupDB.users.email == username),
+                ).to_list()
+                user_groups = [g.id for g in group_q]
+
+                valid_modificaiton = (
+                    (admin and admin_mode)
+                    or (listener.creator and listener.creator.email == username)
+                    or (listener.access.owner == username)
+                    or (username in listener.access.users)
+                    or (not set(user_groups).isdisjoint(listener.access.groups))
+                )
+                if not valid_modificaiton:
+                    raise HTTPException(
+                        status_code=403,
+                        detail=f"User `{username} does not have permission on listener `{listener_id}`",
+                    )
+
             if listener.active is True or (
-                listener.creator and listener.creator.email == current_user
+                listener.creator and listener.creator.email == username
             ):
                 return True
             else:
                 raise HTTPException(
                     status_code=403,
-                    detail=f"User `{current_user} does not have permission on listener `{listener_id}`",
+                    detail=f"User `{username} does not have permission on listener `{listener_id}`",
                 )
         raise HTTPException(status_code=404, detail=f"Listener {listener_id} not found")
 

From 5f85017e94c39a4ea544e8ed84f7695e9a4467b1 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Wed, 22 May 2024 08:59:28 -0500
Subject: [PATCH 21/30] ObjectId -> PydanticObjectId

---
 backend/app/routers/authorization.py        |  2 +-
 backend/app/routers/datasets.py             | 40 ++++++++++++---------
 backend/app/routers/files.py                | 20 +++++++----
 backend/app/routers/folders.py              |  2 +-
 backend/app/routers/groups.py               |  9 ++---
 backend/app/routers/jobs.py                 |  6 ++--
 backend/app/routers/listeners.py            |  2 +-
 backend/app/routers/metadata_datasets.py    | 10 +++---
 backend/app/routers/metadata_files.py       | 22 ++++++------
 backend/app/routers/public_datasets.py      | 26 ++++++++------
 backend/app/routers/public_files.py         | 12 ++++---
 backend/app/routers/public_folders.py       |  2 +-
 backend/app/routers/public_visualization.py |  4 ++-
 backend/app/routers/visualization.py        |  4 ++-
 backend/app/search/index.py                 | 12 +++----
 backend/message_listener.py                 |  2 +-
 16 files changed, 100 insertions(+), 75 deletions(-)

diff --git a/backend/app/routers/authorization.py b/backend/app/routers/authorization.py
index d4ef9e3a7..ff9effd8e 100644
--- a/backend/app/routers/authorization.py
+++ b/backend/app/routers/authorization.py
@@ -346,7 +346,7 @@ async def get_dataset_roles(
         roles = DatasetRoles(dataset_id=str(dataset.id))
 
         async for auth in AuthorizationDB.find(
-            AuthorizationDB.dataset_id == ObjectId(dataset_id)
+            AuthorizationDB.dataset_id == PydanticObjectId(dataset_id)
         ):
             # First, fetch all groups that have a role on the dataset
             group_user_counts = {}
diff --git a/backend/app/routers/datasets.py b/backend/app/routers/datasets.py
index c50ea0170..2dc8400c6 100644
--- a/backend/app/routers/datasets.py
+++ b/backend/app/routers/datasets.py
@@ -292,18 +292,18 @@ async def get_dataset_files(
     if (await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
         if authenticated or public or (admin and admin_mode):
             query = [
-                FileDBViewList.dataset_id == ObjectId(dataset_id),
+                FileDBViewList.dataset_id == PydanticObjectId(dataset_id),
             ]
         else:
             query = [
-                FileDBViewList.dataset_id == ObjectId(dataset_id),
+                FileDBViewList.dataset_id == PydanticObjectId(dataset_id),
                 Or(
                     FileDBViewList.creator.email == user_id,
                     FileDBViewList.auth.user_ids == user_id,
                 ),
             ]
         if folder_id is not None:
-            query.append(FileDBViewList.folder_id == ObjectId(folder_id))
+            query.append(FileDBViewList.folder_id == PydanticObjectId(folder_id))
 
         files_and_count = (
             await FileDBViewList.find(*query)
@@ -366,7 +366,7 @@ async def patch_dataset(
 
         if dataset_info.status is not None:
             query = [
-                FileDBViewList.dataset_id == ObjectId(dataset_id),
+                FileDBViewList.dataset_id == PydanticObjectId(dataset_id),
             ]
             files_views = await FileDBViewList.find(*query).to_list()
             for file_view in files_views:
@@ -454,18 +454,20 @@ async def get_dataset_folders(
     if (await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
         if authenticated or public:
             query = [
-                FolderDBViewList.dataset_id == ObjectId(dataset_id),
+                FolderDBViewList.dataset_id == PydanticObjectId(dataset_id),
             ]
         else:
             query = [
-                FolderDBViewList.dataset_id == ObjectId(dataset_id),
+                FolderDBViewList.dataset_id == PydanticObjectId(dataset_id),
                 Or(
                     FolderDBViewList.creator.email == user_id,
                     FolderDBViewList.auth.user_ids == user_id,
                 ),
             ]
         if parent_folder is not None:
-            query.append(FolderDBViewList.parent_folder == ObjectId(parent_folder))
+            query.append(
+                FolderDBViewList.parent_folder == PydanticObjectId(parent_folder)
+            )
         else:
             query.append(FolderDBViewList.parent_folder == None)  # noqa: E711
 
@@ -506,11 +508,11 @@ async def get_dataset_folders_and_files(
     if (await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
         if authenticated or public or (admin and admin_mode):
             query = [
-                FolderFileViewList.dataset_id == ObjectId(dataset_id),
+                FolderFileViewList.dataset_id == PydanticObjectId(dataset_id),
             ]
         else:
             query = [
-                FolderFileViewList.dataset_id == ObjectId(dataset_id),
+                FolderFileViewList.dataset_id == PydanticObjectId(dataset_id),
                 Or(
                     FolderFileViewList.creator.email == user_id,
                     FolderFileViewList.auth.user_ids == user_id,
@@ -528,8 +530,8 @@ async def get_dataset_folders_and_files(
         else:
             query.append(
                 Or(
-                    FolderFileViewList.folder_id == ObjectId(folder_id),
-                    FolderFileViewList.parent_folder == ObjectId(folder_id),
+                    FolderFileViewList.folder_id == PydanticObjectId(folder_id),
+                    FolderFileViewList.parent_folder == PydanticObjectId(folder_id),
                 )
             )
 
@@ -576,15 +578,17 @@ async def delete_folder(
     if (await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
         if (folder := await FolderDB.get(PydanticObjectId(folder_id))) is not None:
             # delete current folder and files
-            async for file in FileDB.find(FileDB.folder_id == ObjectId(folder_id)):
+            async for file in FileDB.find(
+                FileDB.folder_id == PydanticObjectId(folder_id)
+            ):
                 await remove_file_entry(file.id, fs, es)
 
             # recursively delete child folder and files
             async def _delete_nested_folders(parent_folder_id):
                 while (
                     await FolderDB.find_one(
-                        FolderDB.dataset_id == ObjectId(dataset_id),
-                        FolderDB.parent_folder == ObjectId(parent_folder_id),
+                        FolderDB.dataset_id == PydanticObjectId(dataset_id),
+                        FolderDB.parent_folder == PydanticObjectId(parent_folder_id),
                     )
                 ) is not None:
                     async for subfolder in FolderDB.find(
@@ -916,7 +920,7 @@ async def download_dataset(
 
         # Write dataset metadata if found
         metadata = await MetadataDB.find(
-            MetadataDB.resource.resource_id == ObjectId(dataset_id)
+            MetadataDB.resource.resource_id == PydanticObjectId(dataset_id)
         ).to_list()
         if len(metadata) > 0:
             datasetmetadata_path = os.path.join(
@@ -934,7 +938,9 @@ async def download_dataset(
         bag_size = 0  # bytes
         file_count = 0
 
-        async for file in FileDB.find(FileDB.dataset_id == ObjectId(dataset_id)):
+        async for file in FileDB.find(
+            FileDB.dataset_id == PydanticObjectId(dataset_id)
+        ):
             file_count += 1
             file_name = file.name
             if file.folder_id is not None:
@@ -963,7 +969,7 @@ async def download_dataset(
             bag_size += current_file_size
 
             metadata = await MetadataDB.find(
-                MetadataDB.resource.resource_id == ObjectId(dataset_id)
+                MetadataDB.resource.resource_id == PydanticObjectId(dataset_id)
             ).to_list()
             if len(metadata) > 0:
                 metadata_filename = file_name + "_metadata.json"
diff --git a/backend/app/routers/files.py b/backend/app/routers/files.py
index cb7e61988..4784c3716 100644
--- a/backend/app/routers/files.py
+++ b/backend/app/routers/files.py
@@ -48,7 +48,7 @@ async def _resubmit_file_extractors(
     """
     resubmitted_jobs = []
     async for job in EventListenerJobDB.find(
-        EventListenerJobDB.resource_ref.resource_id == ObjectId(file.id),
+        EventListenerJobDB.resource_ref.resource_id == PydanticObjectId(file.id),
         EventListenerJobDB.resource_ref.version == file.version_num - 1,
     ):
         resubmitted_job = {"listener_id": job.listener_id, "parameters": job.parameters}
@@ -179,8 +179,12 @@ async def remove_file_entry(
     delete_document_by_id(es, settings.elasticsearch_index, str(file_id))
     if (file := await FileDB.get(PydanticObjectId(file_id))) is not None:
         await file.delete()
-    await MetadataDB.find(MetadataDB.resource.resource_id == ObjectId(file_id)).delete()
-    await FileVersionDB.find(FileVersionDB.file_id == ObjectId(file_id)).delete()
+    await MetadataDB.find(
+        MetadataDB.resource.resource_id == PydanticObjectId(file_id)
+    ).delete()
+    await FileVersionDB.find(
+        FileVersionDB.file_id == PydanticObjectId(file_id)
+    ).delete()
 
 
 async def remove_local_file_entry(file_id: Union[str, ObjectId], es: Elasticsearch):
@@ -192,7 +196,9 @@ async def remove_local_file_entry(file_id: Union[str, ObjectId], es: Elasticsear
         # TODO: delete from disk - should this be allowed if Clowder didn't originally write the file?
         # os.path.remove(file.storage_path)
         await file.delete()
-    await MetadataDB.find(MetadataDB.resource.resource_id == ObjectId(file_id)).delete()
+    await MetadataDB.find(
+        MetadataDB.resource.resource_id == PydanticObjectId(file_id)
+    ).delete()
 
 
 @router.put("/{file_id}", response_model=FileOut)
@@ -264,7 +270,7 @@ async def update_file(
 
         # updating metadata in elasticsearch
         metadata = await MetadataDB.find_one(
-            MetadataDB.resource.resource_id == ObjectId(updated_file.id)
+            MetadataDB.resource.resource_id == PydanticObjectId(updated_file.id)
         )
         if metadata:
             doc = {
@@ -299,7 +305,7 @@ async def download_file(
             if version is not None:
                 # Version is specified, so get the minio ID from versions table if possible
                 file_vers = await FileVersionDB.find_one(
-                    FileVersionDB.file_id == ObjectId(file_id),
+                    FileVersionDB.file_id == PydanticObjectId(file_id),
                     FileVersionDB.version_num == version,
                 )
                 if file_vers is not None:
@@ -462,7 +468,7 @@ async def get_file_versions(
         mongo_versions = []
         if file.storage_type == StorageType.MINIO:
             async for ver in (
-                FileVersionDB.find(FileVersionDB.file_id == ObjectId(file_id))
+                FileVersionDB.find(FileVersionDB.file_id == PydanticObjectId(file_id))
                 .sort(-FileVersionDB.created)
                 .skip(skip)
                 .limit(limit)
diff --git a/backend/app/routers/folders.py b/backend/app/routers/folders.py
index 2c9fecd9c..22278dea6 100644
--- a/backend/app/routers/folders.py
+++ b/backend/app/routers/folders.py
@@ -17,7 +17,7 @@ async def download_folder(
         # TODO switch to $graphLookup
         while (
             current_folder := await FolderDB.find_one(
-                FolderDB.id == ObjectId(current_folder_id)
+                FolderDB.id == PydanticObjectId(current_folder_id)
             )
         ) is not None:
             folder_info = {
diff --git a/backend/app/routers/groups.py b/backend/app/routers/groups.py
index e1ce7a3dd..00532f987 100644
--- a/backend/app/routers/groups.py
+++ b/backend/app/routers/groups.py
@@ -185,7 +185,7 @@ async def edit_group(
                 await group.replace()
                 # Add user to all affected Authorization entries
                 await AuthorizationDB.find(
-                    AuthorizationDB.group_ids == ObjectId(group_id),
+                    AuthorizationDB.group_ids == PydanticObjectId(group_id),
                 ).update(
                     Push({AuthorizationDB.user_ids: user.email}),
                 )
@@ -241,7 +241,7 @@ async def add_member(
                 await group.replace()
                 # Add user to all affected Authorization entries
                 await AuthorizationDB.find(
-                    AuthorizationDB.group_ids == ObjectId(group_id),
+                    AuthorizationDB.group_ids == PydanticObjectId(group_id),
                 ).update(
                     Push({AuthorizationDB.user_ids: username}),
                 )
@@ -269,8 +269,9 @@ async def remove_member(
             return group
 
         # Remove user from all affected Authorization entries
-        # TODO not sure if this is right
-        async for auth in AuthorizationDB.find({"group_ids": ObjectId(group_id)}):
+        async for auth in AuthorizationDB.find(
+            AuthorizationDB.group_ids == PydanticObjectId(group_id),
+        ):
             auth.user_ids.remove(username)
             await auth.replace()
 
diff --git a/backend/app/routers/jobs.py b/backend/app/routers/jobs.py
index 35ceb94de..b665e6379 100644
--- a/backend/app/routers/jobs.py
+++ b/backend/app/routers/jobs.py
@@ -68,12 +68,14 @@ async def get_all_job_summary(
     if file_id is not None:
         filters.append(EventListenerJobViewList.resource_ref.collection == "files")
         filters.append(
-            EventListenerJobViewList.resource_ref.resource_id == ObjectId(file_id)
+            EventListenerJobViewList.resource_ref.resource_id
+            == PydanticObjectId(file_id)
         )
     if dataset_id is not None:
         filters.append(EventListenerJobViewList.resource_ref.collection == "datasets")
         filters.append(
-            EventListenerJobViewList.resource_ref.resource_id == ObjectId(dataset_id)
+            EventListenerJobViewList.resource_ref.resource_id
+            == PydanticObjectId(dataset_id)
         )
 
     jobs_and_count = (
diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index e0724a691..c3e35fe75 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -518,7 +518,7 @@ async def _set_active_flag(
         listener_id -- UUID of the listener to be enabled/disabled
     """
     listener = await EventListenerDB.find_one(
-        EventListenerDB.id == ObjectId(listener_id)
+        EventListenerDB.id == PydanticObjectId(listener_id)
     )
     if listener:
         try:
diff --git a/backend/app/routers/metadata_datasets.py b/backend/app/routers/metadata_datasets.py
index 94cf0a886..bb82f1031 100644
--- a/backend/app/routers/metadata_datasets.py
+++ b/backend/app/routers/metadata_datasets.py
@@ -129,7 +129,7 @@ async def replace_dataset_metadata(
         Metadata document that was updated
     """
     if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
-        query = [MetadataDB.resource.resource_id == ObjectId(dataset_id)]
+        query = [MetadataDB.resource.resource_id == PydanticObjectId(dataset_id)]
 
         # Filter by MetadataAgent
         if metadata_in.listener is not None:
@@ -183,7 +183,7 @@ async def update_dataset_metadata(
         Metadata document that was updated
     """
     if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
-        query = [MetadataDB.resource.resource_id == ObjectId(dataset_id)]
+        query = [MetadataDB.resource.resource_id == PydanticObjectId(dataset_id)]
         content = metadata_in.content
 
         if metadata_in.metadata_id is not None:
@@ -246,7 +246,7 @@ async def get_dataset_metadata(
     allow: bool = Depends(Authorization("viewer")),
 ):
     if (await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
-        query = [MetadataDB.resource.resource_id == ObjectId(dataset_id)]
+        query = [MetadataDB.resource.resource_id == PydanticObjectId(dataset_id)]
 
         if listener_name is not None:
             query.append(MetadataDB.agent.listener.name == listener_name)
@@ -278,12 +278,12 @@ async def delete_dataset_metadata(
 ):
     if (await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
         # filter by metadata_id or definition
-        query = [MetadataDB.resource.resource_id == ObjectId(dataset_id)]
+        query = [MetadataDB.resource.resource_id == PydanticObjectId(dataset_id)]
         if metadata_in.metadata_id is not None:
             # If a specific metadata_id is provided, delete the matching entry
             if (
                 await MetadataDB.find_one(
-                    MetadataDB.metadata_id == ObjectId(metadata_in.metadata_id)
+                    MetadataDB.metadata_id == PydanticObjectId(metadata_in.metadata_id)
                 )
             ) is not None:
                 query.append(MetadataDB.metadata_id == metadata_in.metadata_id)
diff --git a/backend/app/routers/metadata_files.py b/backend/app/routers/metadata_files.py
index 0ce5efd65..0c1f8dda1 100644
--- a/backend/app/routers/metadata_files.py
+++ b/backend/app/routers/metadata_files.py
@@ -162,13 +162,13 @@ async def replace_file_metadata(
     """
     if (file := await FileDB.get(PydanticObjectId(file_id))) is not None:
         # First, make sure the metadata we are replacing actually exists.
-        query = [MetadataDB.resource.resource_id == ObjectId(file_id)]
+        query = [MetadataDB.resource.resource_id == PydanticObjectId(file_id)]
 
         version = metadata_in.file_version
         if version is not None:
             if (
                 await FileVersionDB.find_one(
-                    FileVersionDB.file_id == ObjectId(file_id),
+                    FileVersionDB.file_id == PydanticObjectId(file_id),
                     FileVersionDB.version_num == version,
                 )
             ) is None:
@@ -238,7 +238,7 @@ async def update_file_metadata(
     # check if metadata with file version exists, replace metadata if none exists
     if (
         await MetadataDB.find_one(
-            MetadataDB.resource.resource_id == ObjectId(file_id),
+            MetadataDB.resource.resource_id == PydanticObjectId(file_id),
             MetadataDB.resource.version == metadata_in.file_version,
         )
     ) is None:
@@ -246,14 +246,14 @@ async def update_file_metadata(
         return result
 
     if (file := await FileDB.get(PydanticObjectId(file_id))) is not None:
-        query = [MetadataDB.resource.resource_id == ObjectId(file_id)]
+        query = [MetadataDB.resource.resource_id == PydanticObjectId(file_id)]
         content = metadata_in.content
 
         if metadata_in.metadata_id is not None:
             # If a specific metadata_id is provided, validate the patch against existing context
             if (
                 existing_md := await MetadataDB.find_one(
-                    MetadataDB.id == ObjectId(metadata_in.metadata_id)
+                    MetadataDB.id == PydanticObjectId(metadata_in.metadata_id)
                 )
             ) is not None:
                 content = await validate_context(
@@ -273,7 +273,7 @@ async def update_file_metadata(
         if metadata_in.file_version is not None:
             if (
                 await FileVersionDB.find_one(
-                    FileVersionDB.file_id == ObjectId(file_id),
+                    FileVersionDB.file_id == PydanticObjectId(file_id),
                     FileVersionDB.version_num == metadata_in.file_version,
                 )
             ) is None:
@@ -330,14 +330,14 @@ async def get_file_metadata(
 ):
     """Get file metadata."""
     if (file := await FileDB.get(PydanticObjectId(file_id))) is not None:
-        query = [MetadataDB.resource.resource_id == ObjectId(file_id)]
+        query = [MetadataDB.resource.resource_id == PydanticObjectId(file_id)]
 
         # Validate specified version, or use latest by default
         if not all_versions:
             if version is not None and version > 0:
                 if (
                     await FileVersionDB.find_one(
-                        FileVersionDB.file_id == ObjectId(file_id),
+                        FileVersionDB.file_id == PydanticObjectId(file_id),
                         FileVersionDB.version_num == version,
                     )
                 ) is None:
@@ -385,13 +385,13 @@ async def delete_file_metadata(
     allow: bool = Depends(FileAuthorization("editor")),
 ):
     if (await FileDB.get(PydanticObjectId(file_id))) is not None:
-        query = [MetadataDB.resource.resource_id == ObjectId(file_id)]
+        query = [MetadataDB.resource.resource_id == PydanticObjectId(file_id)]
 
         # # Validate specified version, or use latest by default
         # if version is not None:
         #     if (
         #         version_q := await FileVersionDB.find_one(
-        #             FileVersionDB.file_id == ObjectId(file_id),
+        #             FileVersionDB.file_id == PydanticObjectId(file_id),
         #             FileVersionDB.version_num == version,
         #         )
         #     ) is None:
@@ -409,7 +409,7 @@ async def delete_file_metadata(
             # If a specific metadata_id is provided, delete the matching entry
             if (
                 await MetadataDB.find_one(
-                    MetadataDB.metadata_id == ObjectId(metadata_in.metadata_id)
+                    MetadataDB.metadata_id == PydanticObjectId(metadata_in.metadata_id)
                 )
             ) is not None:
                 query.append(MetadataDB.metadata_id == metadata_in.metadata_id)
diff --git a/backend/app/routers/public_datasets.py b/backend/app/routers/public_datasets.py
index 1b31acf78..9aa90223a 100644
--- a/backend/app/routers/public_datasets.py
+++ b/backend/app/routers/public_datasets.py
@@ -86,10 +86,10 @@ async def get_dataset_files(
     if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
         if dataset.status == DatasetStatus.PUBLIC.name:
             query = [
-                FileDBViewList.dataset_id == ObjectId(dataset_id),
+                FileDBViewList.dataset_id == PydanticObjectId(dataset_id),
             ]
             if folder_id is not None:
-                query.append(FileDBViewList.folder_id == ObjectId(folder_id))
+                query.append(FileDBViewList.folder_id == PydanticObjectId(folder_id))
             files = await FileDBViewList.find(*query).skip(skip).limit(limit).to_list()
             return [file.dict() for file in files]
     raise HTTPException(status_code=404, detail=f"Dataset {dataset_id} not found")
@@ -105,10 +105,12 @@ async def get_dataset_folders(
     if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
         if dataset.status == DatasetStatus.PUBLIC.name:
             query = [
-                FolderDBViewList.dataset_id == ObjectId(dataset_id),
+                FolderDBViewList.dataset_id == PydanticObjectId(dataset_id),
             ]
             if parent_folder is not None:
-                query.append(FolderDBViewList.parent_folder == ObjectId(parent_folder))
+                query.append(
+                    FolderDBViewList.parent_folder == PydanticObjectId(parent_folder)
+                )
             else:
                 query.append(FolderDBViewList.parent_folder == None)  # noqa: E711
             folders = (
@@ -128,7 +130,7 @@ async def get_dataset_folders_and_files(
     if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
         if dataset.status == DatasetStatus.PUBLIC.name:
             query = [
-                FolderFileViewList.dataset_id == ObjectId(dataset_id),
+                FolderFileViewList.dataset_id == PydanticObjectId(dataset_id),
             ]
             if folder_id is None:
                 # only show folder and file at root level without parent folder
@@ -141,8 +143,8 @@ async def get_dataset_folders_and_files(
             else:
                 query.append(
                     Or(
-                        FolderFileViewList.folder_id == ObjectId(folder_id),
-                        FolderFileViewList.parent_folder == ObjectId(folder_id),
+                        FolderFileViewList.folder_id == PydanticObjectId(folder_id),
+                        FolderFileViewList.parent_folder == PydanticObjectId(folder_id),
                     )
                 )
 
@@ -188,7 +190,7 @@ async def get_dataset_metadata(
 ):
     if (dataset := await DatasetDB.get(PydanticObjectId(dataset_id))) is not None:
         if dataset.status == DatasetStatus.PUBLIC.name:
-            query = [MetadataDB.resource.resource_id == ObjectId(dataset_id)]
+            query = [MetadataDB.resource.resource_id == PydanticObjectId(dataset_id)]
 
             if listener_name is not None:
                 query.append(MetadataDB.agent.listener.name == listener_name)
@@ -242,7 +244,7 @@ async def download_dataset(
 
             # Write dataset metadata if found
             metadata = await MetadataDB.find(
-                MetadataDB.resource.resource_id == ObjectId(dataset_id)
+                MetadataDB.resource.resource_id == PydanticObjectId(dataset_id)
             ).to_list()
             if len(metadata) > 0:
                 datasetmetadata_path = os.path.join(
@@ -260,7 +262,9 @@ async def download_dataset(
             bag_size = 0  # bytes
             file_count = 0
 
-            async for file in FileDB.find(FileDB.dataset_id == ObjectId(dataset_id)):
+            async for file in FileDB.find(
+                FileDB.dataset_id == PydanticObjectId(dataset_id)
+            ):
                 file_count += 1
                 file_name = file.name
                 if file.folder_id is not None:
@@ -291,7 +295,7 @@ async def download_dataset(
                 bag_size += current_file_size
 
                 metadata = await MetadataDB.find(
-                    MetadataDB.resource.resource_id == ObjectId(dataset_id)
+                    MetadataDB.resource.resource_id == PydanticObjectId(dataset_id)
                 ).to_list()
                 if len(metadata) > 0:
                     metadata_filename = file_name + "_metadata.json"
diff --git a/backend/app/routers/public_files.py b/backend/app/routers/public_files.py
index 9530f8cf1..750e097e1 100644
--- a/backend/app/routers/public_files.py
+++ b/backend/app/routers/public_files.py
@@ -51,7 +51,7 @@ async def get_file_version_details(
         ) is not None:
             if dataset.status == DatasetStatus.PUBLIC.name:
                 file_vers = await FileVersionDB.find_one(
-                    FileVersionDB.file_id == ObjectId(file_id),
+                    FileVersionDB.file_id == PydanticObjectId(file_id),
                     FileVersionDB.version_num == version_num,
                 )
                 file_vers_dict = file_vers.dict()
@@ -78,7 +78,9 @@ async def get_file_versions(
             if dataset.status == DatasetStatus.PUBLIC.name:
                 mongo_versions = []
                 async for ver in (
-                    FileVersionDB.find(FileVersionDB.file_id == ObjectId(file_id))
+                    FileVersionDB.find(
+                        FileVersionDB.file_id == PydanticObjectId(file_id)
+                    )
                     .sort(-FileVersionDB.created)
                     .skip(skip)
                     .limit(limit)
@@ -104,7 +106,7 @@ async def download_file(
                 if version is not None:
                     # Version is specified, so get the minio ID from versions table if possible
                     file_vers = await FileVersionDB.find_one(
-                        FileVersionDB.file_id == ObjectId(file_id),
+                        FileVersionDB.file_id == PydanticObjectId(file_id),
                         FileVersionDB.version_num == version,
                     )
                     if file_vers is not None:
@@ -187,14 +189,14 @@ async def get_file_metadata(
             dataset := await DatasetDB.get(PydanticObjectId(file.dataset_id))
         ) is not None:
             if dataset.status == DatasetStatus.PUBLIC.name:
-                query = [MetadataDB.resource.resource_id == ObjectId(file_id)]
+                query = [MetadataDB.resource.resource_id == PydanticObjectId(file_id)]
 
                 # Validate specified version, or use latest by default
                 if not all_versions:
                     if version is not None:
                         if (
                             await FileVersionDB.find_one(
-                                FileVersionDB.file_id == ObjectId(file_id),
+                                FileVersionDB.file_id == PydanticObjectId(file_id),
                                 FileVersionDB.version_num == version,
                             )
                         ) is None:
diff --git a/backend/app/routers/public_folders.py b/backend/app/routers/public_folders.py
index 2c9fecd9c..22278dea6 100644
--- a/backend/app/routers/public_folders.py
+++ b/backend/app/routers/public_folders.py
@@ -17,7 +17,7 @@ async def download_folder(
         # TODO switch to $graphLookup
         while (
             current_folder := await FolderDB.find_one(
-                FolderDB.id == ObjectId(current_folder_id)
+                FolderDB.id == PydanticObjectId(current_folder_id)
             )
         ) is not None:
             folder_info = {
diff --git a/backend/app/routers/public_visualization.py b/backend/app/routers/public_visualization.py
index b40b4b2ac..610d48661 100644
--- a/backend/app/routers/public_visualization.py
+++ b/backend/app/routers/public_visualization.py
@@ -87,7 +87,9 @@ async def download_visualization_url(
 async def get_resource_visconfig(
     resource_id: PydanticObjectId,
 ):
-    query = [VisualizationConfigDB.resource.resource_id == ObjectId(resource_id)]
+    query = [
+        VisualizationConfigDB.resource.resource_id == PydanticObjectId(resource_id)
+    ]
     visconfigs = []
     async for vzconfig in VisualizationConfigDB.find(*query):
         config_visdata = []
diff --git a/backend/app/routers/visualization.py b/backend/app/routers/visualization.py
index 3e9c13828..5a243c8a0 100644
--- a/backend/app/routers/visualization.py
+++ b/backend/app/routers/visualization.py
@@ -201,7 +201,9 @@ async def get_resource_visconfig(
     resource_id: PydanticObjectId,
     user=Depends(get_current_user),
 ):
-    query = [VisualizationConfigDB.resource.resource_id == ObjectId(resource_id)]
+    query = [
+        VisualizationConfigDB.resource.resource_id == PydanticObjectId(resource_id)
+    ]
     visconfigs = []
     async for vzconfig in VisualizationConfigDB.find(*query):
         config_visdata = []
diff --git a/backend/app/search/index.py b/backend/app/search/index.py
index 18de9fd9e..a1f7e2ba0 100644
--- a/backend/app/search/index.py
+++ b/backend/app/search/index.py
@@ -25,7 +25,7 @@ async def index_dataset(
         # Get authorized users from db
         authorized_user_ids = []
         async for auth in AuthorizationDB.find(
-            AuthorizationDB.dataset_id == ObjectId(dataset.id)
+            AuthorizationDB.dataset_id == PydanticObjectId(dataset.id)
         ):
             authorized_user_ids += auth.user_ids
     else:
@@ -34,7 +34,7 @@ async def index_dataset(
     # Get full metadata from db (granular updates possible but complicated)
     metadata = []
     async for md in MetadataDB.find(
-        MetadataDB.resource.resource_id == ObjectId(dataset.id)
+        MetadataDB.resource.resource_id == PydanticObjectId(dataset.id)
     ):
         metadata.append(md.content)
     dataset_status = dataset.status
@@ -76,7 +76,7 @@ async def index_file(
         # Get authorized users from db
         authorized_user_ids = []
         async for auth in AuthorizationDB.find(
-            AuthorizationDB.dataset_id == ObjectId(file.dataset_id)
+            AuthorizationDB.dataset_id == PydanticObjectId(file.dataset_id)
         ):
             authorized_user_ids += auth.user_ids
     else:
@@ -85,7 +85,7 @@ async def index_file(
     # Get full metadata from db (granular updates possible but complicated)
     metadata = []
     async for md in MetadataDB.find(
-        MetadataDB.resource.resource_id == ObjectId(file.id)
+        MetadataDB.resource.resource_id == PydanticObjectId(file.id)
     ):
         metadata.append(md.content)
 
@@ -137,13 +137,13 @@ async def index_thumbnail(
             # Get authorized users from db
             authorized_user_ids = []
             async for auth in AuthorizationDB.find(
-                AuthorizationDB.dataset_id == ObjectId(dataset_id)
+                AuthorizationDB.dataset_id == PydanticObjectId(dataset_id)
             ):
                 authorized_user_ids += auth.user_ids
             # Get full metadata from db (granular updates possible but complicated)
             metadata = []
             async for md in MetadataDB.find(
-                MetadataDB.resource.resource_id == ObjectId(file.id)
+                MetadataDB.resource.resource_id == PydanticObjectId(file.id)
             ):
                 metadata.append(md.content)
             # Add en entry to the file index
diff --git a/backend/message_listener.py b/backend/message_listener.py
index 795c0b271..9517ddfb3 100644
--- a/backend/message_listener.py
+++ b/backend/message_listener.py
@@ -104,7 +104,7 @@ async def callback(message: AbstractIncomingMessage):
 
         # Check if the job exists, and update if so
         job = await EventListenerJobDB.find_one(
-            EventListenerJobDB.id == ObjectId(job_id)
+            EventListenerJobDB.id == PydanticObjectId(job_id)
         )
         if job:
             # Update existing job with new info

From 3b45835386b538bb2652dc2fe79e6807225f5b0f Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Fri, 7 Jun 2024 14:12:12 -0500
Subject: [PATCH 22/30] add missing comma

---
 frontend/src/actions/listeners.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/frontend/src/actions/listeners.js b/frontend/src/actions/listeners.js
index 5f40b0378..7ceaefe02 100644
--- a/frontend/src/actions/listeners.js
+++ b/frontend/src/actions/listeners.js
@@ -24,7 +24,7 @@ export function fetchListeners(
 			label,
 			aliveOnly,
 			process,
-			dataset_id
+			dataset_id,
 			all
 		)
 			.then((json) => {

From 689510aa0025783f156a7e6083ddf11c53854570 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Mon, 17 Jun 2024 14:01:19 -0500
Subject: [PATCH 23/30] Fix failing tests, codegen

---
 backend/app/models/listeners.py               |   1 -
 backend/app/routers/feeds.py                  |  19 +-
 backend/app/tests/test_feeds.py               |   9 +-
 .../openapi/v2/services/ListenersService.ts   | 914 +++++++++---------
 4 files changed, 482 insertions(+), 461 deletions(-)

diff --git a/backend/app/models/listeners.py b/backend/app/models/listeners.py
index be59738c4..a7c720f61 100644
--- a/backend/app/models/listeners.py
+++ b/backend/app/models/listeners.py
@@ -82,7 +82,6 @@ class EventListenerDB(Document, EventListenerBase):
     modified: datetime = Field(default_factory=datetime.now)
     lastAlive: datetime = None
     alive: Optional[bool] = None
-    alive: Optional[bool] = None
     active: bool = False
     properties: Optional[ExtractorInfo] = None
 
diff --git a/backend/app/routers/feeds.py b/backend/app/routers/feeds.py
index c0380811c..991b3e1ab 100644
--- a/backend/app/routers/feeds.py
+++ b/backend/app/routers/feeds.py
@@ -1,8 +1,10 @@
 from typing import List, Optional
+
 from beanie import PydanticObjectId
 from beanie.operators import Or
 from fastapi import APIRouter, Depends, HTTPException
 from pika.adapters.blocking_connection import BlockingChannel
+
 from app.deps.authorization_deps import ListenerAuthorization
 from app.keycloak_auth import get_current_user, get_current_username
 from app.models.feeds import FeedDB, FeedIn, FeedOut
@@ -11,6 +13,7 @@
 from app.models.listeners import EventListenerDB, FeedListener
 from app.models.users import UserOut
 from app.rabbitmq.listeners import submit_file_job
+from app.routers.authentication import get_admin, get_admin_mode
 from app.search.connect import check_search_result
 
 router = APIRouter()
@@ -141,7 +144,11 @@ async def delete_feed(
 
 @router.post("/{feed_id}/listeners", response_model=FeedOut)
 async def associate_listener(
-    feed_id: str, listener: FeedListener, allow: bool = Depends(ListenerAuthorization())
+    feed_id: str,
+    listener: FeedListener,
+    username=Depends(get_current_username),
+    admin=Depends(get_admin),
+    admin_mode=Depends(get_admin_mode),
 ):
     """Associate an existing Event Listener with a Feed, e.g. so it will be triggered on new Feed results.
 
@@ -149,6 +156,16 @@ async def associate_listener(
         feed_id: Feed that should have new Event Listener associated
         listener: JSON object with "listener_id" field and "automatic" bool field (whether to auto-trigger on new data)
     """
+    # Because we have FeedListener rather than listener_id here, we can't use injection for this
+    allow = ListenerAuthorization().__call__(
+        listener.listener_id, username, admin_mode, admin
+    )
+    if not allow:
+        raise HTTPException(
+            status_code=403,
+            detail=f"User `{username} does not have permission on listener `{listener.listener_id}`",
+        )
+
     if (feed := await FeedDB.get(PydanticObjectId(feed_id))) is not None:
         if (
             await EventListenerDB.get(PydanticObjectId(listener.listener_id))
diff --git a/backend/app/tests/test_feeds.py b/backend/app/tests/test_feeds.py
index ecd1ecbe3..678a9cb0d 100644
--- a/backend/app/tests/test_feeds.py
+++ b/backend/app/tests/test_feeds.py
@@ -1,5 +1,7 @@
 import time
 
+from fastapi.testclient import TestClient
+
 from app.config import settings
 from app.tests.utils import (
     create_dataset,
@@ -7,7 +9,6 @@
     register_v2_listener,
     upload_file,
 )
-from fastapi.testclient import TestClient
 
 
 def test_feeds(client: TestClient, headers: dict):
@@ -23,11 +24,15 @@ def test_feeds(client: TestClient, headers: dict):
     feed_id = response.json().get("id")
 
     # Assign listener to feed
+    jbody = {"listener_id": listener_id, "automatic": True}
+    print(jbody)
     response = client.post(
         f"{settings.API_V2_STR}/feeds/{feed_id}/listeners",
-        json={"listener_id": listener_id, "automatic": True},
+        json=jbody,
         headers=headers,
     )
+    print(response)
+    print(response.json())
     assert response.status_code == 200
 
     # Upload file to trigger the feed
diff --git a/frontend/src/openapi/v2/services/ListenersService.ts b/frontend/src/openapi/v2/services/ListenersService.ts
index f6853ff07..b307d7c3a 100644
--- a/frontend/src/openapi/v2/services/ListenersService.ts
+++ b/frontend/src/openapi/v2/services/ListenersService.ts
@@ -1,481 +1,481 @@
 /* istanbul ignore file */
 /* tslint:disable */
 /* eslint-disable */
-import type {EventListenerIn} from '../models/EventListenerIn';
-import type {EventListenerOut} from '../models/EventListenerOut';
-import type {Paged} from '../models/Paged';
-import type {CancelablePromise} from '../core/CancelablePromise';
-import {request as __request} from '../core/request';
+import type { EventListenerIn } from '../models/EventListenerIn';
+import type { EventListenerOut } from '../models/EventListenerOut';
+import type { Paged } from '../models/Paged';
+import type { CancelablePromise } from '../core/CancelablePromise';
+import { request as __request } from '../core/request';
 
 export class ListenersService {
 
-	/**
-	 * Get Instance Id
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static getInstanceIdApiV2ListenersInstanceGet(): CancelablePromise<any> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/instance`,
-		});
-	}
+    /**
+     * Get Instance Id
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static getInstanceIdApiV2ListenersInstanceGet(): CancelablePromise<any> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/instance`,
+        });
+    }
 
-	/**
-	 * Get Listeners
-	 * Get a list of all Event Listeners in the db.
-	 *
-	 * Arguments:
-	 * skip -- number of initial records to skip (i.e. for pagination)
-	 * limit -- restrict number of records to be returned (i.e. for pagination)
-	 * heartbeat_interval -- number of seconds after which a listener is considered dead
-	 * category -- filter by category has to be exact match
-	 * label -- filter by label has to be exact match
-	 * alive_only -- filter by alive status
-	 * process -- filter by file or dataset type (if specified)
-	 * dataset_id -- restrict to listeners that run on the given dataset or a file within (if not otherwise permitted)
-	 * all -- boolean stating if we want to show all listeners irrespective of admin and admin_mode
-	 * @param skip
-	 * @param limit
-	 * @param heartbeatInterval
-	 * @param category
-	 * @param label
-	 * @param aliveOnly
-	 * @param process
-	 * @param all
-	 * @param datasetId
-	 * @returns Paged Successful Response
-	 * @throws ApiError
-	 */
-	public static getListenersApiV2ListenersGet(
-		skip?: number,
-		limit: number = 2,
-		heartbeatInterval: number = 300,
-		category?: string,
-		label?: string,
-		aliveOnly: boolean = false,
-		process?: string,
-		all: boolean = false,
-		datasetId?: string,
-	): CancelablePromise<Paged> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners`,
-			query: {
-				'skip': skip,
-				'limit': limit,
-				'heartbeat_interval': heartbeatInterval,
-				'category': category,
-				'label': label,
-				'alive_only': aliveOnly,
-				'process': process,
-				'all': all,
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Get Listeners
+     * Get a list of all Event Listeners in the db.
+     *
+     * Arguments:
+     * skip -- number of initial records to skip (i.e. for pagination)
+     * limit -- restrict number of records to be returned (i.e. for pagination)
+     * heartbeat_interval -- number of seconds after which a listener is considered dead
+     * category -- filter by category has to be exact match
+     * label -- filter by label has to be exact match
+     * alive_only -- filter by alive status
+     * process -- filter by file or dataset type (if specified)
+     * dataset_id -- restrict to listeners that run on the given dataset or a file within (if not otherwise permitted)
+     * all -- boolean stating if we want to show all listeners irrespective of admin and admin_mode
+     * @param skip
+     * @param limit
+     * @param heartbeatInterval
+     * @param category
+     * @param label
+     * @param aliveOnly
+     * @param process
+     * @param datasetId
+     * @param all
+     * @returns Paged Successful Response
+     * @throws ApiError
+     */
+    public static getListenersApiV2ListenersGet(
+        skip?: number,
+        limit: number = 2,
+        heartbeatInterval: number = 300,
+        category?: string,
+        label?: string,
+        aliveOnly: boolean = false,
+        process?: string,
+        datasetId?: string,
+        all: boolean = false,
+    ): CancelablePromise<Paged> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners`,
+            query: {
+                'skip': skip,
+                'limit': limit,
+                'heartbeat_interval': heartbeatInterval,
+                'category': category,
+                'label': label,
+                'alive_only': aliveOnly,
+                'process': process,
+                'dataset_id': datasetId,
+                'all': all,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Save Listener
-	 * Register a new Event Listener with the system.
-	 * @param requestBody
-	 * @returns EventListenerOut Successful Response
-	 * @throws ApiError
-	 */
-	public static saveListenerApiV2ListenersPost(
-		requestBody: EventListenerIn,
-	): CancelablePromise<EventListenerOut> {
-		return __request({
-			method: 'POST',
-			path: `/api/v2/listeners`,
-			body: requestBody,
-			mediaType: 'application/json',
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Save Listener
+     * Register a new Event Listener with the system.
+     * @param requestBody
+     * @returns EventListenerOut Successful Response
+     * @throws ApiError
+     */
+    public static saveListenerApiV2ListenersPost(
+        requestBody: EventListenerIn,
+    ): CancelablePromise<EventListenerOut> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners`,
+            body: requestBody,
+            mediaType: 'application/json',
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Search Listeners
-	 * Search all Event Listeners in the db based on text.
-	 *
-	 * Arguments:
-	 * text -- any text matching name or description
-	 * skip -- number of initial records to skip (i.e. for pagination)
-	 * limit -- restrict number of records to be returned (i.e. for pagination)
-	 * @param text
-	 * @param skip
-	 * @param limit
-	 * @param heartbeatInterval
-	 * @param process
-	 * @param datasetId
-	 * @returns Paged Successful Response
-	 * @throws ApiError
-	 */
-	public static searchListenersApiV2ListenersSearchGet(
-		text: string = '',
-		skip?: number,
-		limit: number = 2,
-		heartbeatInterval: number = 300,
-		process?: string,
-		datasetId?: string,
-	): CancelablePromise<Paged> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/search`,
-			query: {
-				'text': text,
-				'skip': skip,
-				'limit': limit,
-				'heartbeat_interval': heartbeatInterval,
-				'process': process,
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Search Listeners
+     * Search all Event Listeners in the db based on text.
+     *
+     * Arguments:
+     * text -- any text matching name or description
+     * skip -- number of initial records to skip (i.e. for pagination)
+     * limit -- restrict number of records to be returned (i.e. for pagination)
+     * @param text
+     * @param skip
+     * @param limit
+     * @param heartbeatInterval
+     * @param process
+     * @param datasetId
+     * @returns Paged Successful Response
+     * @throws ApiError
+     */
+    public static searchListenersApiV2ListenersSearchGet(
+        text: string = '',
+        skip?: number,
+        limit: number = 2,
+        heartbeatInterval: number = 300,
+        process?: string,
+        datasetId?: string,
+    ): CancelablePromise<Paged> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/search`,
+            query: {
+                'text': text,
+                'skip': skip,
+                'limit': limit,
+                'heartbeat_interval': heartbeatInterval,
+                'process': process,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * List Categories
-	 * Get all the distinct categories of registered listeners in the db
-	 * @returns string Successful Response
-	 * @throws ApiError
-	 */
-	public static listCategoriesApiV2ListenersCategoriesGet(): CancelablePromise<Array<string>> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/categories`,
-		});
-	}
+    /**
+     * List Categories
+     * Get all the distinct categories of registered listeners in the db
+     * @returns string Successful Response
+     * @throws ApiError
+     */
+    public static listCategoriesApiV2ListenersCategoriesGet(): CancelablePromise<Array<string>> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/categories`,
+        });
+    }
 
-	/**
-	 * List Default Labels
-	 * Get all the distinct default labels of registered listeners in the db
-	 * @returns string Successful Response
-	 * @throws ApiError
-	 */
-	public static listDefaultLabelsApiV2ListenersDefaultLabelsGet(): CancelablePromise<Array<string>> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/defaultLabels`,
-		});
-	}
+    /**
+     * List Default Labels
+     * Get all the distinct default labels of registered listeners in the db
+     * @returns string Successful Response
+     * @throws ApiError
+     */
+    public static listDefaultLabelsApiV2ListenersDefaultLabelsGet(): CancelablePromise<Array<string>> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/defaultLabels`,
+        });
+    }
 
-	/**
-	 * Get Listener
-	 * Return JSON information about an Event Listener if it exists.
-	 * @param listenerId
-	 * @param datasetId
-	 * @returns EventListenerOut Successful Response
-	 * @throws ApiError
-	 */
-	public static getListenerApiV2ListenersListenerIdGet(
-		listenerId: string,
-		datasetId?: string,
-	): CancelablePromise<EventListenerOut> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/${listenerId}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Get Listener
+     * Return JSON information about an Event Listener if it exists.
+     * @param listenerId
+     * @param datasetId
+     * @returns EventListenerOut Successful Response
+     * @throws ApiError
+     */
+    public static getListenerApiV2ListenersListenerIdGet(
+        listenerId: string,
+        datasetId?: string,
+    ): CancelablePromise<EventListenerOut> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/${listenerId}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Edit Listener
-	 * Update the information about an existing Event Listener..
-	 *
-	 * Arguments:
-	 * listener_id -- UUID of the listener to be udpated
-	 * listener_in -- JSON object including updated information
-	 * @param listenerId
-	 * @param requestBody
-	 * @param datasetId
-	 * @returns EventListenerOut Successful Response
-	 * @throws ApiError
-	 */
-	public static editListenerApiV2ListenersListenerIdPut(
-		listenerId: string,
-		requestBody: EventListenerIn,
-		datasetId?: string,
-	): CancelablePromise<EventListenerOut> {
-		return __request({
-			method: 'PUT',
-			path: `/api/v2/listeners/${listenerId}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			body: requestBody,
-			mediaType: 'application/json',
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Edit Listener
+     * Update the information about an existing Event Listener..
+     *
+     * Arguments:
+     * listener_id -- UUID of the listener to be udpated
+     * listener_in -- JSON object including updated information
+     * @param listenerId
+     * @param requestBody
+     * @param datasetId
+     * @returns EventListenerOut Successful Response
+     * @throws ApiError
+     */
+    public static editListenerApiV2ListenersListenerIdPut(
+        listenerId: string,
+        requestBody: EventListenerIn,
+        datasetId?: string,
+    ): CancelablePromise<EventListenerOut> {
+        return __request({
+            method: 'PUT',
+            path: `/api/v2/listeners/${listenerId}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            body: requestBody,
+            mediaType: 'application/json',
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Delete Listener
-	 * Remove an Event Listener from the database. Will not clear event history for the listener.
-	 * @param listenerId
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static deleteListenerApiV2ListenersListenerIdDelete(
-		listenerId: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'DELETE',
-			path: `/api/v2/listeners/${listenerId}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Delete Listener
+     * Remove an Event Listener from the database. Will not clear event history for the listener.
+     * @param listenerId
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static deleteListenerApiV2ListenersListenerIdDelete(
+        listenerId: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Check Listener Livelihood
-	 * Return JSON information about an Event Listener if it exists.
-	 * @param listenerId
-	 * @param heartbeatInterval
-	 * @param datasetId
-	 * @returns boolean Successful Response
-	 * @throws ApiError
-	 */
-	public static checkListenerLivelihoodApiV2ListenersListenerIdStatusGet(
-		listenerId: string,
-		heartbeatInterval: number = 300,
-		datasetId?: string,
-	): CancelablePromise<boolean> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/${listenerId}/status`,
-			query: {
-				'heartbeat_interval': heartbeatInterval,
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Check Listener Livelihood
+     * Return JSON information about an Event Listener if it exists.
+     * @param listenerId
+     * @param heartbeatInterval
+     * @param datasetId
+     * @returns boolean Successful Response
+     * @throws ApiError
+     */
+    public static checkListenerLivelihoodApiV2ListenersListenerIdStatusGet(
+        listenerId: string,
+        heartbeatInterval: number = 300,
+        datasetId?: string,
+    ): CancelablePromise<boolean> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/${listenerId}/status`,
+            query: {
+                'heartbeat_interval': heartbeatInterval,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Enable Listener
-	 * Enable an Event Listener. Only admins can enable listeners.
-	 *
-	 * Arguments:
-	 * listener_id -- UUID of the listener to be enabled
-	 * @param listenerId
-	 * @param datasetId
-	 * @returns EventListenerOut Successful Response
-	 * @throws ApiError
-	 */
-	public static enableListenerApiV2ListenersListenerIdEnablePut(
-		listenerId: string,
-		datasetId?: string,
-	): CancelablePromise<EventListenerOut> {
-		return __request({
-			method: 'PUT',
-			path: `/api/v2/listeners/${listenerId}/enable`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Enable Listener
+     * Enable an Event Listener. Only admins can enable listeners.
+     *
+     * Arguments:
+     * listener_id -- UUID of the listener to be enabled
+     * @param listenerId
+     * @param datasetId
+     * @returns EventListenerOut Successful Response
+     * @throws ApiError
+     */
+    public static enableListenerApiV2ListenersListenerIdEnablePut(
+        listenerId: string,
+        datasetId?: string,
+    ): CancelablePromise<EventListenerOut> {
+        return __request({
+            method: 'PUT',
+            path: `/api/v2/listeners/${listenerId}/enable`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Disable Listener
-	 * Disable an Event Listener. Only admins can enable listeners.
-	 *
-	 * Arguments:
-	 * listener_id -- UUID of the listener to be enabled
-	 * @param listenerId
-	 * @param datasetId
-	 * @returns EventListenerOut Successful Response
-	 * @throws ApiError
-	 */
-	public static disableListenerApiV2ListenersListenerIdDisablePut(
-		listenerId: string,
-		datasetId?: string,
-	): CancelablePromise<EventListenerOut> {
-		return __request({
-			method: 'PUT',
-			path: `/api/v2/listeners/${listenerId}/disable`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Disable Listener
+     * Disable an Event Listener. Only admins can enable listeners.
+     *
+     * Arguments:
+     * listener_id -- UUID of the listener to be enabled
+     * @param listenerId
+     * @param datasetId
+     * @returns EventListenerOut Successful Response
+     * @throws ApiError
+     */
+    public static disableListenerApiV2ListenersListenerIdDisablePut(
+        listenerId: string,
+        datasetId?: string,
+    ): CancelablePromise<EventListenerOut> {
+        return __request({
+            method: 'PUT',
+            path: `/api/v2/listeners/${listenerId}/disable`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Add User Permission
-	 * @param listenerId
-	 * @param targetUser
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static addUserPermissionApiV2ListenersListenerIdUsersTargetUserPost(
-		listenerId: string,
-		targetUser: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'POST',
-			path: `/api/v2/listeners/${listenerId}/users/${targetUser}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Add User Permission
+     * @param listenerId
+     * @param targetUser
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static addUserPermissionApiV2ListenersListenerIdUsersTargetUserPost(
+        listenerId: string,
+        targetUser: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners/${listenerId}/users/${targetUser}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Remove User Permission
-	 * @param listenerId
-	 * @param targetUser
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static removeUserPermissionApiV2ListenersListenerIdUsersTargetUserDelete(
-		listenerId: string,
-		targetUser: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'DELETE',
-			path: `/api/v2/listeners/${listenerId}/users/${targetUser}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Remove User Permission
+     * @param listenerId
+     * @param targetUser
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static removeUserPermissionApiV2ListenersListenerIdUsersTargetUserDelete(
+        listenerId: string,
+        targetUser: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}/users/${targetUser}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Add Group Permission
-	 * @param listenerId
-	 * @param targetGroup
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static addGroupPermissionApiV2ListenersListenerIdGroupsTargetGroupPost(
-		listenerId: string,
-		targetGroup: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'POST',
-			path: `/api/v2/listeners/${listenerId}/groups/${targetGroup}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Add Group Permission
+     * @param listenerId
+     * @param targetGroup
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static addGroupPermissionApiV2ListenersListenerIdGroupsTargetGroupPost(
+        listenerId: string,
+        targetGroup: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners/${listenerId}/groups/${targetGroup}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Remove Group Permission
-	 * @param listenerId
-	 * @param targetGroup
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static removeGroupPermissionApiV2ListenersListenerIdGroupsTargetGroupDelete(
-		listenerId: string,
-		targetGroup: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'DELETE',
-			path: `/api/v2/listeners/${listenerId}/groups/${targetGroup}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Remove Group Permission
+     * @param listenerId
+     * @param targetGroup
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static removeGroupPermissionApiV2ListenersListenerIdGroupsTargetGroupDelete(
+        listenerId: string,
+        targetGroup: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}/groups/${targetGroup}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Add Dataset Permission
-	 * @param listenerId
-	 * @param targetDataset
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static addDatasetPermissionApiV2ListenersListenerIdDatasetsTargetDatasetPost(
-		listenerId: string,
-		targetDataset: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'POST',
-			path: `/api/v2/listeners/${listenerId}/datasets/${targetDataset}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Add Dataset Permission
+     * @param listenerId
+     * @param targetDataset
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static addDatasetPermissionApiV2ListenersListenerIdDatasetsTargetDatasetPost(
+        listenerId: string,
+        targetDataset: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners/${listenerId}/datasets/${targetDataset}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Remove Dataset Permission
-	 * @param listenerId
-	 * @param targetDataset
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static removeDatasetPermissionApiV2ListenersListenerIdDatasetsTargetDatasetDelete(
-		listenerId: string,
-		targetDataset: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'DELETE',
-			path: `/api/v2/listeners/${listenerId}/datasets/${targetDataset}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Remove Dataset Permission
+     * @param listenerId
+     * @param targetDataset
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static removeDatasetPermissionApiV2ListenersListenerIdDatasetsTargetDatasetDelete(
+        listenerId: string,
+        targetDataset: string,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}/datasets/${targetDataset}`,
+            query: {
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-}
+}
\ No newline at end of file

From ffe054a922dd8c9d18e5a1e2550b46762ef8d76d Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Thu, 11 Jul 2024 07:58:02 -0500
Subject: [PATCH 24/30] import Union

---
 backend/app/routers/files.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/app/routers/files.py b/backend/app/routers/files.py
index 5ab379dca..2c5513f03 100644
--- a/backend/app/routers/files.py
+++ b/backend/app/routers/files.py
@@ -1,7 +1,7 @@
 import io
 import time
 from datetime import datetime, timedelta
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from app import dependencies
 from app.config import settings

From e7afb65969f6e96926060d10b754dbbb526959c7 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Thu, 18 Jul 2024 08:33:25 -0500
Subject: [PATCH 25/30] fix missing import

---
 backend/app/routers/files.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/app/routers/files.py b/backend/app/routers/files.py
index 2c5513f03..9bd6bad2a 100644
--- a/backend/app/routers/files.py
+++ b/backend/app/routers/files.py
@@ -23,7 +23,7 @@
 from app.rabbitmq.listeners import EventListenerJobDB, submit_file_job
 from app.routers.feeds import check_feed_listeners
 from app.routers.utils import get_content_type
-from app.search.connect import insert_record, update_record
+from app.search.connect import insert_record, update_record, delete_document_by_id
 from app.search.index import index_file, index_thumbnail
 from beanie import PydanticObjectId
 from beanie.odm.operators.find.logical import Or

From 901e317793be75961b398543a18874b7f9ddf236 Mon Sep 17 00:00:00 2001
From: Max Burnette <mburnet2@illinois.edu>
Date: Fri, 30 Aug 2024 14:13:46 -0500
Subject: [PATCH 26/30] Update ListenersService.ts

---
 .../openapi/v2/services/ListenersService.ts   | 980 +++++++++---------
 1 file changed, 499 insertions(+), 481 deletions(-)

diff --git a/frontend/src/openapi/v2/services/ListenersService.ts b/frontend/src/openapi/v2/services/ListenersService.ts
index 765ec2099..f2aca35e4 100644
--- a/frontend/src/openapi/v2/services/ListenersService.ts
+++ b/frontend/src/openapi/v2/services/ListenersService.ts
@@ -1,505 +1,523 @@
 /* istanbul ignore file */
 /* tslint:disable */
 /* eslint-disable */
-import type {EventListenerIn} from '../models/EventListenerIn';
-import type {EventListenerOut} from '../models/EventListenerOut';
-import type {Paged} from '../models/Paged';
-import type {CancelablePromise} from '../core/CancelablePromise';
-import {request as __request} from '../core/request';
+import type { EventListenerIn } from '../models/EventListenerIn';
+import type { EventListenerOut } from '../models/EventListenerOut';
+import type { Paged } from '../models/Paged';
+import type { CancelablePromise } from '../core/CancelablePromise';
+import { request as __request } from '../core/request';
 
 export class ListenersService {
 
-	/**
-	 * Get Instance Id
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static getInstanceIdApiV2ListenersInstanceGet(): CancelablePromise<any> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/instance`,
-		});
-	}
+    /**
+     * Get Instance Id
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static getInstanceIdApiV2ListenersInstanceGet(): CancelablePromise<any> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/instance`,
+        });
+    }
 
-	/**
-	 * Get Listeners
-	 * Get a list of all Event Listeners in the db.
-	 *
-	 * Arguments:
-	 * skip -- number of initial records to skip (i.e. for pagination)
-	 * limit -- restrict number of records to be returned (i.e. for pagination)
-	 * heartbeat_interval -- number of seconds after which a listener is considered dead
-	 * category -- filter by category has to be exact match
-	 * label -- filter by label has to be exact match
-	 * alive_only -- filter by alive status
-	 * process -- filter by file or dataset type (if specified)
-	 * dataset_id -- restrict to listeners that run on the given dataset or a file within (if not otherwise permitted)
-	 * all -- boolean stating if we want to show all listeners irrespective of admin and admin_mode
-	 * @param skip
-	 * @param limit
-	 * @param heartbeatInterval
-	 * @param category
-	 * @param label
-	 * @param aliveOnly
-	 * @param process
-	 * @param all
-	 * @param enableAdmin
-	 * @param datasetId
-	 * @returns Paged Successful Response
-	 * @throws ApiError
-	 */
-	public static getListenersApiV2ListenersGet(
-		skip?: number,
-		limit: number = 2,
-		heartbeatInterval: number = 300,
-		category?: string,
-		label?: string,
-		aliveOnly: boolean = false,
-		process?: string,
-		all: boolean = false,
-		enableAdmin: boolean = false,
-		datasetId?: string,
-	): CancelablePromise<Paged> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners`,
-			query: {
-				'skip': skip,
-				'limit': limit,
-				'heartbeat_interval': heartbeatInterval,
-				'category': category,
-				'label': label,
-				'alive_only': aliveOnly,
-				'process': process,
-				'all': all,
-				'enable_admin': enableAdmin,
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Get Listeners
+     * Get a list of all Event Listeners in the db.
+     *
+     * Arguments:
+     * skip -- number of initial records to skip (i.e. for pagination)
+     * limit -- restrict number of records to be returned (i.e. for pagination)
+     * heartbeat_interval -- number of seconds after which a listener is considered dead
+     * category -- filter by category has to be exact match
+     * label -- filter by label has to be exact match
+     * alive_only -- filter by alive status
+     * process -- filter by file or dataset type (if specified)
+     * dataset_id -- restrict to listeners that run on the given dataset or a file within (if not otherwise permitted)
+     * all -- boolean stating if we want to show all listeners irrespective of admin and admin_mode
+     * @param skip
+     * @param limit
+     * @param heartbeatInterval
+     * @param category
+     * @param label
+     * @param aliveOnly
+     * @param process
+     * @param datasetId
+     * @param all
+     * @param enableAdmin
+     * @returns Paged Successful Response
+     * @throws ApiError
+     */
+    public static getListenersApiV2ListenersGet(
+        skip?: number,
+        limit: number = 2,
+        heartbeatInterval: number = 300,
+        category?: string,
+        label?: string,
+        aliveOnly: boolean = false,
+        process?: string,
+        datasetId?: string,
+        all: boolean = false,
+        enableAdmin: boolean = false,
+    ): CancelablePromise<Paged> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners`,
+            query: {
+                'skip': skip,
+                'limit': limit,
+                'heartbeat_interval': heartbeatInterval,
+                'category': category,
+                'label': label,
+                'alive_only': aliveOnly,
+                'process': process,
+                'dataset_id': datasetId,
+                'all': all,
+                'enable_admin': enableAdmin,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Save Listener
-	 * Register a new Event Listener with the system.
-	 * @param requestBody
-	 * @returns EventListenerOut Successful Response
-	 * @throws ApiError
-	 */
-	public static saveListenerApiV2ListenersPost(
-		requestBody: EventListenerIn,
-	): CancelablePromise<EventListenerOut> {
-		return __request({
-			method: 'POST',
-			path: `/api/v2/listeners`,
-			body: requestBody,
-			mediaType: 'application/json',
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Save Listener
+     * Register a new Event Listener with the system.
+     * @param requestBody
+     * @returns EventListenerOut Successful Response
+     * @throws ApiError
+     */
+    public static saveListenerApiV2ListenersPost(
+        requestBody: EventListenerIn,
+    ): CancelablePromise<EventListenerOut> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners`,
+            body: requestBody,
+            mediaType: 'application/json',
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Search Listeners
-	 * Search all Event Listeners in the db based on text.
-	 *
-	 * Arguments:
-	 * text -- any text matching name or description
-	 * skip -- number of initial records to skip (i.e. for pagination)
-	 * limit -- restrict number of records to be returned (i.e. for pagination)
-	 * @param text
-	 * @param skip
-	 * @param limit
-	 * @param heartbeatInterval
-	 * @param process
-	 * @param enableAdmin
-	 * @param datasetId
-	 * @returns Paged Successful Response
-	 * @throws ApiError
-	 */
-	public static searchListenersApiV2ListenersSearchGet(
-		text: string = '',
-		skip?: number,
-		limit: number = 2,
-		heartbeatInterval: number = 300,
-		process?: string,
-		enableAdmin: boolean = false,
-		datasetId?: string,
-	): CancelablePromise<Paged> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/search`,
-			query: {
-				'text': text,
-				'skip': skip,
-				'limit': limit,
-				'heartbeat_interval': heartbeatInterval,
-				'process': process,
-				'enable_admin': enableAdmin,
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Search Listeners
+     * Search all Event Listeners in the db based on text.
+     *
+     * Arguments:
+     * text -- any text matching name or description
+     * skip -- number of initial records to skip (i.e. for pagination)
+     * limit -- restrict number of records to be returned (i.e. for pagination)
+     * @param text
+     * @param skip
+     * @param limit
+     * @param heartbeatInterval
+     * @param process
+     * @param datasetId
+     * @param enableAdmin
+     * @returns Paged Successful Response
+     * @throws ApiError
+     */
+    public static searchListenersApiV2ListenersSearchGet(
+        text: string = '',
+        skip?: number,
+        limit: number = 2,
+        heartbeatInterval: number = 300,
+        process?: string,
+        datasetId?: string,
+        enableAdmin: boolean = false,
+    ): CancelablePromise<Paged> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/search`,
+            query: {
+                'text': text,
+                'skip': skip,
+                'limit': limit,
+                'heartbeat_interval': heartbeatInterval,
+                'process': process,
+                'dataset_id': datasetId,
+                'enable_admin': enableAdmin,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * List Categories
-	 * Get all the distinct categories of registered listeners in the db
-	 * @returns string Successful Response
-	 * @throws ApiError
-	 */
-	public static listCategoriesApiV2ListenersCategoriesGet(): CancelablePromise<Array<string>> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/categories`,
-		});
-	}
+    /**
+     * List Categories
+     * Get all the distinct categories of registered listeners in the db
+     * @returns string Successful Response
+     * @throws ApiError
+     */
+    public static listCategoriesApiV2ListenersCategoriesGet(): CancelablePromise<Array<string>> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/categories`,
+        });
+    }
 
-	/**
-	 * List Default Labels
-	 * Get all the distinct default labels of registered listeners in the db
-	 * @returns string Successful Response
-	 * @throws ApiError
-	 */
-	public static listDefaultLabelsApiV2ListenersDefaultLabelsGet(): CancelablePromise<Array<string>> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/defaultLabels`,
-		});
-	}
+    /**
+     * List Default Labels
+     * Get all the distinct default labels of registered listeners in the db
+     * @returns string Successful Response
+     * @throws ApiError
+     */
+    public static listDefaultLabelsApiV2ListenersDefaultLabelsGet(): CancelablePromise<Array<string>> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/defaultLabels`,
+        });
+    }
 
-	/**
-	 * Get Listener
-	 * Return JSON information about an Event Listener if it exists.
-	 * @param listenerId
-	 * @param enableAdmin
-	 * @param datasetId
-	 * @returns EventListenerOut Successful Response
-	 * @throws ApiError
-	 */
-	public static getListenerApiV2ListenersListenerIdGet(
-		listenerId: string,
-		enableAdmin: boolean = false,
-		datasetId?: string,
-	): CancelablePromise<EventListenerOut> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/${listenerId}`,
-			query: {
-				'enable_admin': enableAdmin,
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Get Listener
+     * Return JSON information about an Event Listener if it exists.
+     * @param listenerId
+     * @param enableAdmin
+     * @param datasetId
+     * @returns EventListenerOut Successful Response
+     * @throws ApiError
+     */
+    public static getListenerApiV2ListenersListenerIdGet(
+        listenerId: string,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<EventListenerOut> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/${listenerId}`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Edit Listener
-	 * Update the information about an existing Event Listener..
-	 *
-	 * Arguments:
-	 * listener_id -- UUID of the listener to be udpated
-	 * listener_in -- JSON object including updated information
-	 * @param listenerId
-	 * @param requestBody
-	 * @param enableAdmin
-	 * @param datasetId
-	 * @returns EventListenerOut Successful Response
-	 * @throws ApiError
-	 */
-	public static editListenerApiV2ListenersListenerIdPut(
-		listenerId: string,
-		requestBody: EventListenerIn,
-		enableAdmin: boolean = false,
-		datasetId?: string,
-	): CancelablePromise<EventListenerOut> {
-		return __request({
-			method: 'PUT',
-			path: `/api/v2/listeners/${listenerId}`,
-			query: {
-				'enable_admin': enableAdmin,
-				'dataset_id': datasetId,
-			},
-			body: requestBody,
-			mediaType: 'application/json',
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Edit Listener
+     * Update the information about an existing Event Listener..
+     *
+     * Arguments:
+     * listener_id -- UUID of the listener to be udpated
+     * listener_in -- JSON object including updated information
+     * @param listenerId
+     * @param requestBody
+     * @param enableAdmin
+     * @param datasetId
+     * @returns EventListenerOut Successful Response
+     * @throws ApiError
+     */
+    public static editListenerApiV2ListenersListenerIdPut(
+        listenerId: string,
+        requestBody: EventListenerIn,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<EventListenerOut> {
+        return __request({
+            method: 'PUT',
+            path: `/api/v2/listeners/${listenerId}`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            body: requestBody,
+            mediaType: 'application/json',
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Delete Listener
-	 * Remove an Event Listener from the database. Will not clear event history for the listener.
-	 * @param listenerId
-	 * @param enableAdmin
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static deleteListenerApiV2ListenersListenerIdDelete(
-		listenerId: string,
-		enableAdmin: boolean = false,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'DELETE',
-			path: `/api/v2/listeners/${listenerId}`,
-			query: {
-				'enable_admin': enableAdmin,
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Delete Listener
+     * Remove an Event Listener from the database. Will not clear event history for the listener.
+     * @param listenerId
+     * @param enableAdmin
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static deleteListenerApiV2ListenersListenerIdDelete(
+        listenerId: string,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Check Listener Livelihood
-	 * Return JSON information about an Event Listener if it exists.
-	 * @param listenerId
-	 * @param heartbeatInterval
-	 * @param enableAdmin
-	 * @param datasetId
-	 * @returns boolean Successful Response
-	 * @throws ApiError
-	 */
-	public static checkListenerLivelihoodApiV2ListenersListenerIdStatusGet(
-		listenerId: string,
-		heartbeatInterval: number = 300,
-		enableAdmin: boolean = false,
-		datasetId?: string,
-	): CancelablePromise<boolean> {
-		return __request({
-			method: 'GET',
-			path: `/api/v2/listeners/${listenerId}/status`,
-			query: {
-				'heartbeat_interval': heartbeatInterval,
-				'enable_admin': enableAdmin,
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Check Listener Livelihood
+     * Return JSON information about an Event Listener if it exists.
+     * @param listenerId
+     * @param heartbeatInterval
+     * @param enableAdmin
+     * @param datasetId
+     * @returns boolean Successful Response
+     * @throws ApiError
+     */
+    public static checkListenerLivelihoodApiV2ListenersListenerIdStatusGet(
+        listenerId: string,
+        heartbeatInterval: number = 300,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<boolean> {
+        return __request({
+            method: 'GET',
+            path: `/api/v2/listeners/${listenerId}/status`,
+            query: {
+                'heartbeat_interval': heartbeatInterval,
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Enable Listener
-	 * Enable an Event Listener. Only admins can enable listeners.
-	 *
-	 * Arguments:
-	 * listener_id -- UUID of the listener to be enabled
-	 * @param listenerId
-	 * @param enableAdmin
-	 * @param datasetId
-	 * @returns EventListenerOut Successful Response
-	 * @throws ApiError
-	 */
-	public static enableListenerApiV2ListenersListenerIdEnablePut(
-		listenerId: string,
-		enableAdmin: boolean = false,
-		datasetId?: string,
-	): CancelablePromise<EventListenerOut> {
-		return __request({
-			method: 'PUT',
-			path: `/api/v2/listeners/${listenerId}/enable`,
-			query: {
-				'enable_admin': enableAdmin,
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Enable Listener
+     * Enable an Event Listener. Only admins can enable listeners.
+     *
+     * Arguments:
+     * listener_id -- UUID of the listener to be enabled
+     * @param listenerId
+     * @param enableAdmin
+     * @param datasetId
+     * @returns EventListenerOut Successful Response
+     * @throws ApiError
+     */
+    public static enableListenerApiV2ListenersListenerIdEnablePut(
+        listenerId: string,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<EventListenerOut> {
+        return __request({
+            method: 'PUT',
+            path: `/api/v2/listeners/${listenerId}/enable`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Disable Listener
-	 * Disable an Event Listener. Only admins can enable listeners.
-	 *
-	 * Arguments:
-	 * listener_id -- UUID of the listener to be enabled
-	 * @param listenerId
-	 * @param enableAdmin
-	 * @param datasetId
-	 * @returns EventListenerOut Successful Response
-	 * @throws ApiError
-	 */
-	public static disableListenerApiV2ListenersListenerIdDisablePut(
-		listenerId: string,
-		enableAdmin: boolean = false,
-		datasetId?: string,
-	): CancelablePromise<EventListenerOut> {
-		return __request({
-			method: 'PUT',
-			path: `/api/v2/listeners/${listenerId}/disable`,
-			query: {
-				'enable_admin': enableAdmin,
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Disable Listener
+     * Disable an Event Listener. Only admins can enable listeners.
+     *
+     * Arguments:
+     * listener_id -- UUID of the listener to be enabled
+     * @param listenerId
+     * @param enableAdmin
+     * @param datasetId
+     * @returns EventListenerOut Successful Response
+     * @throws ApiError
+     */
+    public static disableListenerApiV2ListenersListenerIdDisablePut(
+        listenerId: string,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<EventListenerOut> {
+        return __request({
+            method: 'PUT',
+            path: `/api/v2/listeners/${listenerId}/disable`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Add User Permission
-	 * @param listenerId
-	 * @param targetUser
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static addUserPermissionApiV2ListenersListenerIdUsersTargetUserPost(
-		listenerId: string,
-		targetUser: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'POST',
-			path: `/api/v2/listeners/${listenerId}/users/${targetUser}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Add User Permission
+     * @param listenerId
+     * @param targetUser
+     * @param enableAdmin
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static addUserPermissionApiV2ListenersListenerIdUsersTargetUserPost(
+        listenerId: string,
+        targetUser: string,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners/${listenerId}/users/${targetUser}`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Remove User Permission
-	 * @param listenerId
-	 * @param targetUser
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static removeUserPermissionApiV2ListenersListenerIdUsersTargetUserDelete(
-		listenerId: string,
-		targetUser: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'DELETE',
-			path: `/api/v2/listeners/${listenerId}/users/${targetUser}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Remove User Permission
+     * @param listenerId
+     * @param targetUser
+     * @param enableAdmin
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static removeUserPermissionApiV2ListenersListenerIdUsersTargetUserDelete(
+        listenerId: string,
+        targetUser: string,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}/users/${targetUser}`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Add Group Permission
-	 * @param listenerId
-	 * @param targetGroup
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static addGroupPermissionApiV2ListenersListenerIdGroupsTargetGroupPost(
-		listenerId: string,
-		targetGroup: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'POST',
-			path: `/api/v2/listeners/${listenerId}/groups/${targetGroup}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Add Group Permission
+     * @param listenerId
+     * @param targetGroup
+     * @param enableAdmin
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static addGroupPermissionApiV2ListenersListenerIdGroupsTargetGroupPost(
+        listenerId: string,
+        targetGroup: string,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners/${listenerId}/groups/${targetGroup}`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Remove Group Permission
-	 * @param listenerId
-	 * @param targetGroup
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static removeGroupPermissionApiV2ListenersListenerIdGroupsTargetGroupDelete(
-		listenerId: string,
-		targetGroup: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'DELETE',
-			path: `/api/v2/listeners/${listenerId}/groups/${targetGroup}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Remove Group Permission
+     * @param listenerId
+     * @param targetGroup
+     * @param enableAdmin
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static removeGroupPermissionApiV2ListenersListenerIdGroupsTargetGroupDelete(
+        listenerId: string,
+        targetGroup: string,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}/groups/${targetGroup}`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Add Dataset Permission
-	 * @param listenerId
-	 * @param targetDataset
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static addDatasetPermissionApiV2ListenersListenerIdDatasetsTargetDatasetPost(
-		listenerId: string,
-		targetDataset: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'POST',
-			path: `/api/v2/listeners/${listenerId}/datasets/${targetDataset}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Add Dataset Permission
+     * @param listenerId
+     * @param targetDataset
+     * @param enableAdmin
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static addDatasetPermissionApiV2ListenersListenerIdDatasetsTargetDatasetPost(
+        listenerId: string,
+        targetDataset: string,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'POST',
+            path: `/api/v2/listeners/${listenerId}/datasets/${targetDataset}`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-	/**
-	 * Remove Dataset Permission
-	 * @param listenerId
-	 * @param targetDataset
-	 * @param datasetId
-	 * @returns any Successful Response
-	 * @throws ApiError
-	 */
-	public static removeDatasetPermissionApiV2ListenersListenerIdDatasetsTargetDatasetDelete(
-		listenerId: string,
-		targetDataset: string,
-		datasetId?: string,
-	): CancelablePromise<any> {
-		return __request({
-			method: 'DELETE',
-			path: `/api/v2/listeners/${listenerId}/datasets/${targetDataset}`,
-			query: {
-				'dataset_id': datasetId,
-			},
-			errors: {
-				422: `Validation Error`,
-			},
-		});
-	}
+    /**
+     * Remove Dataset Permission
+     * @param listenerId
+     * @param targetDataset
+     * @param enableAdmin
+     * @param datasetId
+     * @returns any Successful Response
+     * @throws ApiError
+     */
+    public static removeDatasetPermissionApiV2ListenersListenerIdDatasetsTargetDatasetDelete(
+        listenerId: string,
+        targetDataset: string,
+        enableAdmin: boolean = false,
+        datasetId?: string,
+    ): CancelablePromise<any> {
+        return __request({
+            method: 'DELETE',
+            path: `/api/v2/listeners/${listenerId}/datasets/${targetDataset}`,
+            query: {
+                'enable_admin': enableAdmin,
+                'dataset_id': datasetId,
+            },
+            errors: {
+                422: `Validation Error`,
+            },
+        });
+    }
 
-}
+}
\ No newline at end of file

From 71fd9c837c46442cac321779d9fee72daced5d4a Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Sat, 31 Aug 2024 12:41:59 -0500
Subject: [PATCH 27/30] fixing an error i was seeing that left the listener
 page totally blank dataset_id was being sent in by the component as 'true'
 when it should be 'null' and then the 'true' should have gone after

---
 frontend/src/components/listeners/AllListeners.tsx | 1 +
 1 file changed, 1 insertion(+)

diff --git a/frontend/src/components/listeners/AllListeners.tsx b/frontend/src/components/listeners/AllListeners.tsx
index 6d97535df..b9eef54a2 100644
--- a/frontend/src/components/listeners/AllListeners.tsx
+++ b/frontend/src/components/listeners/AllListeners.tsx
@@ -65,6 +65,7 @@ export function AllListeners(props: ListenerProps) {
 				selectedLabel,
 				aliveOnly,
 				process,
+				null,
 				true
 			)
 		);

From 30046bac90a17bcbcf371fb6dfe3258b27a103b3 Mon Sep 17 00:00:00 2001
From: toddn <tcnichol@illinois.edu>
Date: Sat, 31 Aug 2024 14:27:25 -0500
Subject: [PATCH 28/30] fixing registration of extractors

---
 backend/heartbeat_listener.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/heartbeat_listener.py b/backend/heartbeat_listener.py
index 2ed31e01f..6c82f6cbb 100644
--- a/backend/heartbeat_listener.py
+++ b/backend/heartbeat_listener.py
@@ -32,7 +32,7 @@ async def callback(message: AbstractIncomingMessage):
 
         extractor_info = msg["extractor_info"]
         owner = msg["owner"]
-        if owner is not None:
+        if owner is not None and owner != "":
             # Extractor name should match queue, which includes secret key with common extractor_info["name"]
             orig_properties = ExtractorInfo(**extractor_info)
             extractor_name = msg["queue"]

From bee998fdf9f95df1a460d42a453658c8130de8e6 Mon Sep 17 00:00:00 2001
From: Luigi Marini <lmarini@illinois.edu>
Date: Fri, 13 Sep 2024 14:42:19 -0500
Subject: [PATCH 29/30] Fixed listener query when checking if a user is in the
 user list. The query was not matching a user since it was an `equality` query
 on an `array` and not a `in` query.

---
 backend/app/routers/listeners.py |  17 +-
 openapi.json                     | 768 ++++++++++++++++++++++++++-----
 2 files changed, 656 insertions(+), 129 deletions(-)

diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index 933436f72..a1179fdc5 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -4,11 +4,6 @@
 import string
 from typing import List, Optional
 
-from beanie import PydanticObjectId
-from beanie.operators import Or, RegEx, In, NE
-from fastapi import APIRouter, Depends, HTTPException
-from packaging import version
-
 from app.config import settings
 from app.deps.authorization_deps import ListenerAuthorization
 from app.keycloak_auth import get_current_user, get_current_username, get_user
@@ -27,6 +22,10 @@
 from app.models.users import UserOut
 from app.routers.authentication import get_admin, get_admin_mode
 from app.routers.feeds import disassociate_listener_db
+from beanie import PydanticObjectId
+from beanie.operators import In, Or, RegEx
+from fastapi import APIRouter, Depends, HTTPException
+from packaging import version
 
 router = APIRouter()
 legacy_router = APIRouter()  # for back-compatibilty with v1 extractors
@@ -266,7 +265,7 @@ async def search_listeners(
                 Or(
                     EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
-                    EventListenerDB.access.users == user_id,
+                    In(EventListenerDB.access.users, user_id),
                     In(EventListenerDB.access.groups, user_groups),
                 )
             )
@@ -275,7 +274,7 @@ async def search_listeners(
                 Or(
                     EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
-                    EventListenerDB.access.users == user_id,
+                    In(EventListenerDB.access.users, user_id),
                     In(EventListenerDB.access.groups, user_groups),
                     EventListenerDB.access.datasets == PydanticObjectId(dataset_id),
                 )
@@ -423,7 +422,7 @@ async def get_listeners(
                 Or(
                     EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
-                    EventListenerDB.access.users == user_id,
+                    In(EventListenerDB.access.users, user_id),
                     In(EventListenerDB.access.groups, user_groups),
                 )
             )
@@ -432,7 +431,7 @@ async def get_listeners(
                 Or(
                     EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
-                    EventListenerDB.access.users == user_id,
+                    In(EventListenerDB.access.users, user_id),
                     In(EventListenerDB.access.groups, user_groups),
                     EventListenerDB.access.datasets == PydanticObjectId(dataset_id),
                 )
diff --git a/openapi.json b/openapi.json
index 74418d8de..2f1283f74 100644
--- a/openapi.json
+++ b/openapi.json
@@ -7279,7 +7279,7 @@
           "listeners"
         ],
         "summary": "Get Listeners",
-        "description": "Get a list of all Event Listeners in the db.\n\nArguments:\n    skip -- number of initial records to skip (i.e. for pagination)\n    limit -- restrict number of records to be returned (i.e. for pagination)\n    heartbeat_interval -- number of seconds after which a listener is considered dead\n    category -- filter by category has to be exact match\n    label -- filter by label has to be exact match\n    alive_only -- filter by alive status\n    all -- boolean stating if we want to show all listeners irrespective of admin and admin_mode",
+        "description": "Get a list of all Event Listeners in the db.\n\nArguments:\n    skip -- number of initial records to skip (i.e. for pagination)\n    limit -- restrict number of records to be returned (i.e. for pagination)\n    heartbeat_interval -- number of seconds after which a listener is considered dead\n    category -- filter by category has to be exact match\n    label -- filter by label has to be exact match\n    alive_only -- filter by alive status\n    process -- filter by file or dataset type (if specified)\n    dataset_id -- restrict to listeners that run on the given dataset or a file within (if not otherwise permitted)\n    all -- boolean stating if we want to show all listeners irrespective of admin and admin_mode",
         "operationId": "get_listeners_api_v2_listeners_get",
         "parameters": [
           {
@@ -7352,30 +7352,30 @@
           {
             "required": false,
             "schema": {
-              "title": "All",
-              "type": "boolean",
-              "default": false
+              "title": "Dataset Id",
+              "type": "string"
             },
-            "name": "all",
+            "name": "dataset_id",
             "in": "query"
           },
           {
             "required": false,
             "schema": {
-              "title": "Enable Admin",
+              "title": "All",
               "type": "boolean",
               "default": false
             },
-            "name": "enable_admin",
+            "name": "all",
             "in": "query"
           },
           {
             "required": false,
             "schema": {
-              "title": "Dataset Id",
-              "type": "string"
+              "title": "Enable Admin",
+              "type": "boolean",
+              "default": false
             },
-            "name": "dataset_id",
+            "name": "enable_admin",
             "in": "query"
           }
         ],
@@ -7515,13 +7515,467 @@
             "in": "query"
           },
           {
-            "required": false,
+            "required": false,
+            "schema": {
+              "title": "Process",
+              "type": "string"
+            },
+            "name": "process",
+            "in": "query"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Dataset Id",
+              "type": "string"
+            },
+            "name": "dataset_id",
+            "in": "query"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Enable Admin",
+              "type": "boolean",
+              "default": false
+            },
+            "name": "enable_admin",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful Response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/Paged"
+                }
+              }
+            }
+          },
+          "422": {
+            "description": "Validation Error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/HTTPValidationError"
+                }
+              }
+            }
+          }
+        },
+        "security": [
+          {
+            "OAuth2AuthorizationCodeBearer": []
+          },
+          {
+            "APIKeyHeader": []
+          },
+          {
+            "APIKeyCookie": []
+          }
+        ]
+      }
+    },
+    "/api/v2/listeners/categories": {
+      "get": {
+        "tags": [
+          "listeners"
+        ],
+        "summary": "List Categories",
+        "description": "Get all the distinct categories of registered listeners in the db",
+        "operationId": "list_categories_api_v2_listeners_categories_get",
+        "responses": {
+          "200": {
+            "description": "Successful Response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "title": "Response List Categories Api V2 Listeners Categories Get",
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "security": [
+          {
+            "OAuth2AuthorizationCodeBearer": []
+          },
+          {
+            "APIKeyHeader": []
+          },
+          {
+            "APIKeyCookie": []
+          }
+        ]
+      }
+    },
+    "/api/v2/listeners/defaultLabels": {
+      "get": {
+        "tags": [
+          "listeners"
+        ],
+        "summary": "List Default Labels",
+        "description": "Get all the distinct default labels of registered listeners in the db",
+        "operationId": "list_default_labels_api_v2_listeners_defaultLabels_get",
+        "responses": {
+          "200": {
+            "description": "Successful Response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "title": "Response List Default Labels Api V2 Listeners Defaultlabels Get",
+                  "type": "array",
+                  "items": {
+                    "type": "string"
+                  }
+                }
+              }
+            }
+          }
+        },
+        "security": [
+          {
+            "OAuth2AuthorizationCodeBearer": []
+          },
+          {
+            "APIKeyHeader": []
+          },
+          {
+            "APIKeyCookie": []
+          }
+        ]
+      }
+    },
+    "/api/v2/listeners/{listener_id}": {
+      "get": {
+        "tags": [
+          "listeners"
+        ],
+        "summary": "Get Listener",
+        "description": "Return JSON information about an Event Listener if it exists.",
+        "operationId": "get_listener_api_v2_listeners__listener_id__get",
+        "parameters": [
+          {
+            "required": true,
+            "schema": {
+              "title": "Listener Id",
+              "type": "string"
+            },
+            "name": "listener_id",
+            "in": "path"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Enable Admin",
+              "type": "boolean",
+              "default": false
+            },
+            "name": "enable_admin",
+            "in": "query"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Dataset Id",
+              "type": "string"
+            },
+            "name": "dataset_id",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful Response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/EventListenerOut"
+                }
+              }
+            }
+          },
+          "422": {
+            "description": "Validation Error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/HTTPValidationError"
+                }
+              }
+            }
+          }
+        },
+        "security": [
+          {
+            "OAuth2AuthorizationCodeBearer": []
+          },
+          {
+            "APIKeyHeader": []
+          },
+          {
+            "APIKeyCookie": []
+          }
+        ]
+      },
+      "put": {
+        "tags": [
+          "listeners"
+        ],
+        "summary": "Edit Listener",
+        "description": "Update the information about an existing Event Listener..\n\nArguments:\n    listener_id -- UUID of the listener to be udpated\n    listener_in -- JSON object including updated information",
+        "operationId": "edit_listener_api_v2_listeners__listener_id__put",
+        "parameters": [
+          {
+            "required": true,
+            "schema": {
+              "title": "Listener Id",
+              "type": "string"
+            },
+            "name": "listener_id",
+            "in": "path"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Enable Admin",
+              "type": "boolean",
+              "default": false
+            },
+            "name": "enable_admin",
+            "in": "query"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Dataset Id",
+              "type": "string"
+            },
+            "name": "dataset_id",
+            "in": "query"
+          }
+        ],
+        "requestBody": {
+          "content": {
+            "application/json": {
+              "schema": {
+                "$ref": "#/components/schemas/EventListenerIn"
+              }
+            }
+          },
+          "required": true
+        },
+        "responses": {
+          "200": {
+            "description": "Successful Response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/EventListenerOut"
+                }
+              }
+            }
+          },
+          "422": {
+            "description": "Validation Error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/HTTPValidationError"
+                }
+              }
+            }
+          }
+        },
+        "security": [
+          {
+            "OAuth2AuthorizationCodeBearer": []
+          },
+          {
+            "APIKeyHeader": []
+          },
+          {
+            "APIKeyCookie": []
+          }
+        ]
+      },
+      "delete": {
+        "tags": [
+          "listeners"
+        ],
+        "summary": "Delete Listener",
+        "description": "Remove an Event Listener from the database. Will not clear event history for the listener.",
+        "operationId": "delete_listener_api_v2_listeners__listener_id__delete",
+        "parameters": [
+          {
+            "required": true,
+            "schema": {
+              "title": "Listener Id",
+              "type": "string"
+            },
+            "name": "listener_id",
+            "in": "path"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Enable Admin",
+              "type": "boolean",
+              "default": false
+            },
+            "name": "enable_admin",
+            "in": "query"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Dataset Id",
+              "type": "string"
+            },
+            "name": "dataset_id",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful Response",
+            "content": {
+              "application/json": {
+                "schema": {}
+              }
+            }
+          },
+          "422": {
+            "description": "Validation Error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/HTTPValidationError"
+                }
+              }
+            }
+          }
+        },
+        "security": [
+          {
+            "OAuth2AuthorizationCodeBearer": []
+          },
+          {
+            "APIKeyHeader": []
+          },
+          {
+            "APIKeyCookie": []
+          }
+        ]
+      }
+    },
+    "/api/v2/listeners/{listener_id}/status": {
+      "get": {
+        "tags": [
+          "listeners"
+        ],
+        "summary": "Check Listener Livelihood",
+        "description": "Return JSON information about an Event Listener if it exists.",
+        "operationId": "check_listener_livelihood_api_v2_listeners__listener_id__status_get",
+        "parameters": [
+          {
+            "required": true,
+            "schema": {
+              "title": "Listener Id",
+              "type": "string"
+            },
+            "name": "listener_id",
+            "in": "path"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Heartbeat Interval",
+              "type": "integer",
+              "default": 300
+            },
+            "name": "heartbeat_interval",
+            "in": "query"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Enable Admin",
+              "type": "boolean",
+              "default": false
+            },
+            "name": "enable_admin",
+            "in": "query"
+          },
+          {
+            "required": false,
+            "schema": {
+              "title": "Dataset Id",
+              "type": "string"
+            },
+            "name": "dataset_id",
+            "in": "query"
+          }
+        ],
+        "responses": {
+          "200": {
+            "description": "Successful Response",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "title": "Response Check Listener Livelihood Api V2 Listeners  Listener Id  Status Get",
+                  "type": "boolean"
+                }
+              }
+            }
+          },
+          "422": {
+            "description": "Validation Error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/HTTPValidationError"
+                }
+              }
+            }
+          }
+        },
+        "security": [
+          {
+            "OAuth2AuthorizationCodeBearer": []
+          },
+          {
+            "APIKeyHeader": []
+          },
+          {
+            "APIKeyCookie": []
+          }
+        ]
+      }
+    },
+    "/api/v2/listeners/{listener_id}/enable": {
+      "put": {
+        "tags": [
+          "listeners"
+        ],
+        "summary": "Enable Listener",
+        "description": "Enable an Event Listener. Only admins can enable listeners.\n\nArguments:\n    listener_id -- UUID of the listener to be enabled",
+        "operationId": "enable_listener_api_v2_listeners__listener_id__enable_put",
+        "parameters": [
+          {
+            "required": true,
             "schema": {
-              "title": "Process",
+              "title": "Listener Id",
               "type": "string"
             },
-            "name": "process",
-            "in": "query"
+            "name": "listener_id",
+            "in": "path"
           },
           {
             "required": false,
@@ -7549,7 +8003,7 @@
             "content": {
               "application/json": {
                 "schema": {
-                  "$ref": "#/components/schemas/Paged"
+                  "$ref": "#/components/schemas/EventListenerOut"
                 }
               }
             }
@@ -7578,62 +8032,61 @@
         ]
       }
     },
-    "/api/v2/listeners/categories": {
-      "get": {
+    "/api/v2/listeners/{listener_id}/disable": {
+      "put": {
         "tags": [
           "listeners"
         ],
-        "summary": "List Categories",
-        "description": "Get all the distinct categories of registered listeners in the db",
-        "operationId": "list_categories_api_v2_listeners_categories_get",
-        "responses": {
-          "200": {
-            "description": "Successful Response",
-            "content": {
-              "application/json": {
-                "schema": {
-                  "title": "Response List Categories Api V2 Listeners Categories Get",
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
-                }
-              }
-            }
-          }
-        },
-        "security": [
+        "summary": "Disable Listener",
+        "description": "Disable an Event Listener. Only admins can enable listeners.\n\nArguments:\n    listener_id -- UUID of the listener to be enabled",
+        "operationId": "disable_listener_api_v2_listeners__listener_id__disable_put",
+        "parameters": [
           {
-            "OAuth2AuthorizationCodeBearer": []
+            "required": true,
+            "schema": {
+              "title": "Listener Id",
+              "type": "string"
+            },
+            "name": "listener_id",
+            "in": "path"
           },
           {
-            "APIKeyHeader": []
+            "required": false,
+            "schema": {
+              "title": "Enable Admin",
+              "type": "boolean",
+              "default": false
+            },
+            "name": "enable_admin",
+            "in": "query"
           },
           {
-            "APIKeyCookie": []
+            "required": false,
+            "schema": {
+              "title": "Dataset Id",
+              "type": "string"
+            },
+            "name": "dataset_id",
+            "in": "query"
           }
-        ]
-      }
-    },
-    "/api/v2/listeners/defaultLabels": {
-      "get": {
-        "tags": [
-          "listeners"
         ],
-        "summary": "List Default Labels",
-        "description": "Get all the distinct default labels of registered listeners in the db",
-        "operationId": "list_default_labels_api_v2_listeners_defaultLabels_get",
         "responses": {
           "200": {
             "description": "Successful Response",
             "content": {
               "application/json": {
                 "schema": {
-                  "title": "Response List Default Labels Api V2 Listeners Defaultlabels Get",
-                  "type": "array",
-                  "items": {
-                    "type": "string"
-                  }
+                  "$ref": "#/components/schemas/EventListenerOut"
+                }
+              }
+            }
+          },
+          "422": {
+            "description": "Validation Error",
+            "content": {
+              "application/json": {
+                "schema": {
+                  "$ref": "#/components/schemas/HTTPValidationError"
                 }
               }
             }
@@ -7652,14 +8105,13 @@
         ]
       }
     },
-    "/api/v2/listeners/{listener_id}": {
-      "get": {
+    "/api/v2/listeners/{listener_id}/users/{target_user}": {
+      "post": {
         "tags": [
           "listeners"
         ],
-        "summary": "Get Listener",
-        "description": "Return JSON information about an Event Listener if it exists.",
-        "operationId": "get_listener_api_v2_listeners__listener_id__get",
+        "summary": "Add User Permission",
+        "operationId": "add_user_permission_api_v2_listeners__listener_id__users__target_user__post",
         "parameters": [
           {
             "required": true,
@@ -7670,6 +8122,15 @@
             "name": "listener_id",
             "in": "path"
           },
+          {
+            "required": true,
+            "schema": {
+              "title": "Target User",
+              "type": "string"
+            },
+            "name": "target_user",
+            "in": "path"
+          },
           {
             "required": false,
             "schema": {
@@ -7695,9 +8156,7 @@
             "description": "Successful Response",
             "content": {
               "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/EventListenerOut"
-                }
+                "schema": {}
               }
             }
           },
@@ -7724,13 +8183,12 @@
           }
         ]
       },
-      "put": {
+      "delete": {
         "tags": [
           "listeners"
         ],
-        "summary": "Edit Listener",
-        "description": "Update the information about an existing Event Listener..\n\nArguments:\n    listener_id -- UUID of the listener to be udpated\n    listener_in -- JSON object including updated information",
-        "operationId": "edit_listener_api_v2_listeners__listener_id__put",
+        "summary": "Remove User Permission",
+        "operationId": "remove_user_permission_api_v2_listeners__listener_id__users__target_user__delete",
         "parameters": [
           {
             "required": true,
@@ -7741,6 +8199,15 @@
             "name": "listener_id",
             "in": "path"
           },
+          {
+            "required": true,
+            "schema": {
+              "title": "Target User",
+              "type": "string"
+            },
+            "name": "target_user",
+            "in": "path"
+          },
           {
             "required": false,
             "schema": {
@@ -7761,24 +8228,12 @@
             "in": "query"
           }
         ],
-        "requestBody": {
-          "content": {
-            "application/json": {
-              "schema": {
-                "$ref": "#/components/schemas/EventListenerIn"
-              }
-            }
-          },
-          "required": true
-        },
         "responses": {
           "200": {
             "description": "Successful Response",
             "content": {
               "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/EventListenerOut"
-                }
+                "schema": {}
               }
             }
           },
@@ -7804,14 +8259,15 @@
             "APIKeyCookie": []
           }
         ]
-      },
-      "delete": {
+      }
+    },
+    "/api/v2/listeners/{listener_id}/groups/{target_group}": {
+      "post": {
         "tags": [
           "listeners"
         ],
-        "summary": "Delete Listener",
-        "description": "Remove an Event Listener from the database. Will not clear event history for the listener.",
-        "operationId": "delete_listener_api_v2_listeners__listener_id__delete",
+        "summary": "Add Group Permission",
+        "operationId": "add_group_permission_api_v2_listeners__listener_id__groups__target_group__post",
         "parameters": [
           {
             "required": true,
@@ -7822,6 +8278,15 @@
             "name": "listener_id",
             "in": "path"
           },
+          {
+            "required": true,
+            "schema": {
+              "title": "Target Group",
+              "type": "string"
+            },
+            "name": "target_group",
+            "in": "path"
+          },
           {
             "required": false,
             "schema": {
@@ -7873,16 +8338,13 @@
             "APIKeyCookie": []
           }
         ]
-      }
-    },
-    "/api/v2/listeners/{listener_id}/status": {
-      "get": {
+      },
+      "delete": {
         "tags": [
           "listeners"
         ],
-        "summary": "Check Listener Livelihood",
-        "description": "Return JSON information about an Event Listener if it exists.",
-        "operationId": "check_listener_livelihood_api_v2_listeners__listener_id__status_get",
+        "summary": "Remove Group Permission",
+        "operationId": "remove_group_permission_api_v2_listeners__listener_id__groups__target_group__delete",
         "parameters": [
           {
             "required": true,
@@ -7894,14 +8356,13 @@
             "in": "path"
           },
           {
-            "required": false,
+            "required": true,
             "schema": {
-              "title": "Heartbeat Interval",
-              "type": "integer",
-              "default": 300
+              "title": "Target Group",
+              "type": "string"
             },
-            "name": "heartbeat_interval",
-            "in": "query"
+            "name": "target_group",
+            "in": "path"
           },
           {
             "required": false,
@@ -7928,10 +8389,7 @@
             "description": "Successful Response",
             "content": {
               "application/json": {
-                "schema": {
-                  "title": "Response Check Listener Livelihood Api V2 Listeners  Listener Id  Status Get",
-                  "type": "boolean"
-                }
+                "schema": {}
               }
             }
           },
@@ -7959,14 +8417,13 @@
         ]
       }
     },
-    "/api/v2/listeners/{listener_id}/enable": {
-      "put": {
+    "/api/v2/listeners/{listener_id}/datasets/{target_dataset}": {
+      "post": {
         "tags": [
           "listeners"
         ],
-        "summary": "Enable Listener",
-        "description": "Enable an Event Listener. Only admins can enable listeners.\n\nArguments:\n    listener_id -- UUID of the listener to be enabled",
-        "operationId": "enable_listener_api_v2_listeners__listener_id__enable_put",
+        "summary": "Add Dataset Permission",
+        "operationId": "add_dataset_permission_api_v2_listeners__listener_id__datasets__target_dataset__post",
         "parameters": [
           {
             "required": true,
@@ -7977,6 +8434,15 @@
             "name": "listener_id",
             "in": "path"
           },
+          {
+            "required": true,
+            "schema": {
+              "title": "Target Dataset",
+              "type": "string"
+            },
+            "name": "target_dataset",
+            "in": "path"
+          },
           {
             "required": false,
             "schema": {
@@ -8002,9 +8468,7 @@
             "description": "Successful Response",
             "content": {
               "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/EventListenerOut"
-                }
+                "schema": {}
               }
             }
           },
@@ -8030,16 +8494,13 @@
             "APIKeyCookie": []
           }
         ]
-      }
-    },
-    "/api/v2/listeners/{listener_id}/disable": {
-      "put": {
+      },
+      "delete": {
         "tags": [
           "listeners"
         ],
-        "summary": "Disable Listener",
-        "description": "Disable an Event Listener. Only admins can enable listeners.\n\nArguments:\n    listener_id -- UUID of the listener to be enabled",
-        "operationId": "disable_listener_api_v2_listeners__listener_id__disable_put",
+        "summary": "Remove Dataset Permission",
+        "operationId": "remove_dataset_permission_api_v2_listeners__listener_id__datasets__target_dataset__delete",
         "parameters": [
           {
             "required": true,
@@ -8050,6 +8511,15 @@
             "name": "listener_id",
             "in": "path"
           },
+          {
+            "required": true,
+            "schema": {
+              "title": "Target Dataset",
+              "type": "string"
+            },
+            "name": "target_dataset",
+            "in": "path"
+          },
           {
             "required": false,
             "schema": {
@@ -8075,9 +8545,7 @@
             "description": "Successful Response",
             "content": {
               "application/json": {
-                "schema": {
-                  "$ref": "#/components/schemas/EventListenerOut"
-                }
+                "schema": {}
               }
             }
           },
@@ -11627,6 +12095,52 @@
   },
   "components": {
     "schemas": {
+      "AccessList": {
+        "title": "AccessList",
+        "required": [
+          "owner"
+        ],
+        "type": "object",
+        "properties": {
+          "owner": {
+            "title": "Owner",
+            "type": "string"
+          },
+          "users": {
+            "title": "Users",
+            "type": "array",
+            "items": {
+              "type": "string"
+            },
+            "default": []
+          },
+          "groups": {
+            "title": "Groups",
+            "type": "array",
+            "items": {
+              "type": "string",
+              "examples": [
+                "5eb7cf5a86d9755df3a6c593",
+                "5eb7cfb05e32e07750a1756a"
+              ]
+            },
+            "default": []
+          },
+          "datasets": {
+            "title": "Datasets",
+            "type": "array",
+            "items": {
+              "type": "string",
+              "examples": [
+                "5eb7cf5a86d9755df3a6c593",
+                "5eb7cfb05e32e07750a1756a"
+              ]
+            },
+            "default": []
+          }
+        },
+        "description": "Container object for lists of user emails/group IDs/dataset IDs that can submit to listener.\nThe singular owner is the primary who can modify other lists."
+      },
       "AuthorizationBase": {
         "title": "AuthorizationBase",
         "required": [
@@ -12229,6 +12743,9 @@
             "title": "Description",
             "type": "string",
             "default": ""
+          },
+          "access": {
+            "$ref": "#/components/schemas/AccessList"
           }
         },
         "description": "On submission, minimum info for a listener is name, version and description. Clowder will use name and version to locate queue."
@@ -12420,6 +12937,9 @@
             "type": "string",
             "default": ""
           },
+          "access": {
+            "$ref": "#/components/schemas/AccessList"
+          },
           "id": {
             "title": "Id",
             "type": "string",
@@ -12565,6 +13085,10 @@
             "title": "Version",
             "type": "string",
             "default": "1.0"
+          },
+          "unique_key": {
+            "title": "Unique Key",
+            "type": "string"
           }
         },
         "description": "Currently for extractor_info JSON from Clowder v1 extractors for use with to /api/extractors endpoint."
@@ -13195,6 +13719,10 @@
             "type": "string",
             "default": "1.0"
           },
+          "unique_key": {
+            "title": "Unique Key",
+            "type": "string"
+          },
           "description": {
             "title": "Description",
             "type": "string",

From b4e82e3ede9c61a7681067470f3657202107bd05 Mon Sep 17 00:00:00 2001
From: Luigi Marini <lmarini@illinois.edu>
Date: Fri, 13 Sep 2024 15:29:12 -0500
Subject: [PATCH 30/30] Reverting prior listener user matching query since it
 was not the issue and it broke the query. Not sure what happened.

---
 backend/app/routers/listeners.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/backend/app/routers/listeners.py b/backend/app/routers/listeners.py
index a1179fdc5..06a98367d 100644
--- a/backend/app/routers/listeners.py
+++ b/backend/app/routers/listeners.py
@@ -265,7 +265,7 @@ async def search_listeners(
                 Or(
                     EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
-                    In(EventListenerDB.access.users, user_id),
+                    EventListenerDB.access.users == user_id,
                     In(EventListenerDB.access.groups, user_groups),
                 )
             )
@@ -274,7 +274,7 @@ async def search_listeners(
                 Or(
                     EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
-                    In(EventListenerDB.access.users, user_id),
+                    EventListenerDB.access.users == user_id,
                     In(EventListenerDB.access.groups, user_groups),
                     EventListenerDB.access.datasets == PydanticObjectId(dataset_id),
                 )
@@ -422,7 +422,7 @@ async def get_listeners(
                 Or(
                     EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
-                    In(EventListenerDB.access.users, user_id),
+                    EventListenerDB.access.users == user_id,
                     In(EventListenerDB.access.groups, user_groups),
                 )
             )
@@ -431,7 +431,7 @@ async def get_listeners(
                 Or(
                     EventListenerDB.access == None,
                     EventListenerDB.access.owner == user_id,
-                    In(EventListenerDB.access.users, user_id),
+                    EventListenerDB.access.users == user_id,
                     In(EventListenerDB.access.groups, user_groups),
                     EventListenerDB.access.datasets == PydanticObjectId(dataset_id),
                 )