From e6fbe936e4310655bc2a70be3e4390b87dbdcb92 Mon Sep 17 00:00:00 2001 From: Garrett Michael Flynn Date: Thu, 30 May 2024 08:55:58 -0700 Subject: [PATCH] Fix parallel progress updates and improve naming --- .../core/components/utils/progress.js | 4 +-- src/pyflask/manageNeuroconv/__init__.py | 3 ++- src/pyflask/manageNeuroconv/info/__init__.py | 2 +- src/pyflask/manageNeuroconv/info/sse.py | 25 +------------------ .../manageNeuroconv/manage_neuroconv.py | 4 +-- src/pyflask/namespaces/neuroconv.py | 20 +++++++-------- 6 files changed, 17 insertions(+), 41 deletions(-) diff --git a/src/electron/frontend/core/components/utils/progress.js b/src/electron/frontend/core/components/utils/progress.js index ea0678d12d..b938de34af 100644 --- a/src/electron/frontend/core/components/utils/progress.js +++ b/src/electron/frontend/core/components/utils/progress.js @@ -88,7 +88,7 @@ export const createProgressPopup = async (options, tqdmCallback) => { return { ...commonReturnValue, id, close }; }; -const eventsURL = new URL("/neuroconv/events", baseUrl).href; +const progressEventsUrl = new URL("/neuroconv/events/progress", baseUrl).href; class ProgressHandler { constructor(props) { @@ -112,4 +112,4 @@ class ProgressHandler { removeEventListener = (...args) => this.source.removeEventListener(...args); } -export const progressHandler = new ProgressHandler({ url: eventsURL }); +export const progressHandler = new ProgressHandler({ url: progressEventsUrl }); diff --git a/src/pyflask/manageNeuroconv/__init__.py b/src/pyflask/manageNeuroconv/__init__.py index f9f55e0826..3f618ad797 100644 --- a/src/pyflask/manageNeuroconv/__init__.py +++ b/src/pyflask/manageNeuroconv/__init__.py @@ -12,10 +12,11 @@ inspect_multiple_filesystem_objects, inspect_nwb_file, inspect_nwb_folder, - listen_to_neuroconv_events, + listen_to_neuroconv_progress_events, locate_data, upload_folder_to_dandi, upload_multiple_filesystem_objects_to_dandi, upload_project_to_dandi, validate_metadata, + progress_handler ) diff --git a/src/pyflask/manageNeuroconv/info/__init__.py b/src/pyflask/manageNeuroconv/info/__init__.py index 3a29439d24..74e4eb6ef9 100644 --- a/src/pyflask/manageNeuroconv/info/__init__.py +++ b/src/pyflask/manageNeuroconv/info/__init__.py @@ -1,4 +1,4 @@ -from .sse import announcer, format_sse +from .sse import format_sse from .urls import ( CONVERSION_SAVE_FOLDER_PATH, GUIDE_ROOT_FOLDER, diff --git a/src/pyflask/manageNeuroconv/info/sse.py b/src/pyflask/manageNeuroconv/info/sse.py index b9593cba59..165d2a2293 100644 --- a/src/pyflask/manageNeuroconv/info/sse.py +++ b/src/pyflask/manageNeuroconv/info/sse.py @@ -1,30 +1,7 @@ import json -import queue - def format_sse(data: str, event=None) -> str: msg = f"data: {json.dumps(data)}\n\n" if event is not None: msg = f"event: {event}\n{msg}" - return msg - - -class MessageAnnouncer: - def __init__(self): - self.listeners = [] - - def listen(self): - q = queue.Queue(maxsize=5) - self.listeners.append(q) - return q - - def announce(self, msg, event=None): - msg = format_sse(msg, event) - for i in reversed(range(len(self.listeners))): - try: - self.listeners[i].put_nowait(msg) - except queue.Full: - del self.listeners[i] - - -announcer = MessageAnnouncer() + return msg \ No newline at end of file diff --git a/src/pyflask/manageNeuroconv/manage_neuroconv.py b/src/pyflask/manageNeuroconv/manage_neuroconv.py index 3cbf9ab0f4..ab3ba7dc24 100644 --- a/src/pyflask/manageNeuroconv/manage_neuroconv.py +++ b/src/pyflask/manageNeuroconv/manage_neuroconv.py @@ -732,7 +732,7 @@ def convert_to_nwb(info: dict) -> str: options = ( { interface: ( - {"stub_test": info["stub_test"]} # , "iter_opts": {"report_hook": update_conversion_progress}} + {"stub_test": info["stub_test"]} if available_options.get("properties").get(interface).get("properties", {}).get("stub_test") else {} ) @@ -911,7 +911,7 @@ def upload_project_to_dandi( # Create an events endpoint -def listen_to_neuroconv_events(): +def listen_to_neuroconv_progress_events(): messages = progress_handler.listen() # returns a queue.Queue while True: msg = messages.get() # blocks until a new message arrives diff --git a/src/pyflask/namespaces/neuroconv.py b/src/pyflask/namespaces/neuroconv.py index e3383fde3b..db2027387a 100644 --- a/src/pyflask/namespaces/neuroconv.py +++ b/src/pyflask/namespaces/neuroconv.py @@ -13,14 +13,14 @@ inspect_multiple_filesystem_objects, inspect_nwb_file, inspect_nwb_folder, - listen_to_neuroconv_events, + listen_to_neuroconv_progress_events, locate_data, upload_folder_to_dandi, upload_multiple_filesystem_objects_to_dandi, upload_project_to_dandi, validate_metadata, + progress_handler ) -from manageNeuroconv.info import announcer neuroconv_namespace = Namespace("neuroconv", description="Neuroconv neuroconv_namespace for the NWB GUIDE.") @@ -158,17 +158,16 @@ def post(self): class InspectNWBFolder(Resource): @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): - url = f"{request.url_root}neuroconv/announce" + url = f"{request.url_root}neuroconv/announce/progress" return inspect_nwb_folder(url, neuroconv_namespace.payload) -@neuroconv_namespace.route("/announce") +@neuroconv_namespace.route("/announce/progress") class InspectNWBFolder(Resource): @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def post(self): data = neuroconv_namespace.payload - announcer.announce(data) - + progress_handler.announce(data) return True @@ -178,7 +177,7 @@ class InspectNWBFolder(Resource): def post(self): from os.path import isfile - url = f"{request.url_root}neuroconv/announce" + url = f"{request.url_root}neuroconv/announce/progress" paths = neuroconv_namespace.payload["paths"] @@ -207,9 +206,8 @@ def post(self): # Create an events endpoint -# announcer.announce('test', 'publish') -@neuroconv_namespace.route("/events", methods=["GET"]) -class Events(Resource): +@neuroconv_namespace.route("/events/progress", methods=["GET"]) +class ProgressEvents(Resource): @neuroconv_namespace.doc(responses={200: "Success", 400: "Bad Request", 500: "Internal server error"}) def get(self): - return Response(listen_to_neuroconv_events(), mimetype="text/event-stream") + return Response(listen_to_neuroconv_progress_events(), mimetype="text/event-stream")