Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplified parallel structure #57

Merged
merged 43 commits into from
Apr 17, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
5969b4d
add docstrings and annotations
Apr 8, 2024
9a368cc
debug
Apr 8, 2024
3d9ab79
debugging
Apr 8, 2024
0724613
swap from try/except; enhance verbosity
Apr 8, 2024
14e2a55
explain locality
Apr 8, 2024
babf1ac
simplify main runner
Apr 8, 2024
07d5984
Update src/tqdm_publisher/_demos/_parallel_bars/_server.py
CodyCBakerPhD Apr 8, 2024
d4be732
GF updates
garrettmflynn Apr 8, 2024
1d90a0e
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 8, 2024
44dd169
Update _handler.py
garrettmflynn Apr 8, 2024
06c701b
Merge branch 'doc_improvements' of https://github.com/catalystneuro/t…
garrettmflynn Apr 8, 2024
b6b5ea5
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 8, 2024
27afa89
Show as much metadata as possible from format_dict
garrettmflynn Apr 8, 2024
9079b18
Merge branch 'doc_improvements' into metadata-visualization-improvement
garrettmflynn Apr 8, 2024
4cb2b06
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 8, 2024
920ad34
add docs and simplify logic
CodyCBakerPhD Apr 9, 2024
54f4950
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 9, 2024
65b5ef1
Merge branch 'main' into doc_improvements
CodyCBakerPhD Apr 9, 2024
f634c4f
Merge branch 'doc_improvements' into metadata-visualization-improvement
CodyCBakerPhD Apr 9, 2024
b501b3e
Update _server.py
garrettmflynn Apr 9, 2024
c474dfa
Update _server.py
garrettmflynn Apr 9, 2024
8058bdb
Update _server.py
garrettmflynn Apr 9, 2024
91857a3
change to public method and enhance docstring
CodyCBakerPhD Apr 10, 2024
81e34ad
simplify logic
CodyCBakerPhD Apr 10, 2024
2443e87
Fix id references
garrettmflynn Apr 10, 2024
0106b58
Update _server.py
garrettmflynn Apr 10, 2024
9240235
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 10, 2024
0d4341f
Merge branch 'doc_improvements' into metadata-visualization-improvement
garrettmflynn Apr 10, 2024
788b784
Update elements.js
garrettmflynn Apr 10, 2024
19a7886
Fix client
garrettmflynn Apr 10, 2024
add05c6
Merge branch 'doc_improvements' into metadata-visualization-improvement
garrettmflynn Apr 10, 2024
1b26552
Remove WS demo for parallel demos
garrettmflynn Apr 10, 2024
1e5b039
Merge branch 'main' into metadata-visualization-improvement
garrettmflynn Apr 10, 2024
7627082
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 10, 2024
3857508
Request initial state immediately
garrettmflynn Apr 13, 2024
4594d38
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 13, 2024
95a3c27
Update elements.js
garrettmflynn Apr 14, 2024
03a2259
Merge branch 'metadata-visualization-improvement' of https://github.c…
garrettmflynn Apr 14, 2024
34a51eb
Update elements.js
garrettmflynn Apr 14, 2024
d167e49
Simplified parallel structure
garrettmflynn Apr 15, 2024
49f828b
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 15, 2024
54734ca
Update _server.py
garrettmflynn Apr 16, 2024
db58fa0
Merge branch 'main' into simplified-parallel-structure
garrettmflynn Apr 17, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
62 changes: 0 additions & 62 deletions src/tqdm_publisher/_demos/_parallel_bars/_client.py

This file was deleted.

74 changes: 21 additions & 53 deletions src/tqdm_publisher/_demos/_parallel_bars/_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,21 +3,16 @@
import asyncio
import json
import sys
import threading
import time
import uuid
from concurrent.futures import ProcessPoolExecutor, as_completed
from typing import List, Union
from typing import List

import requests
from flask import Flask, Response, jsonify, request
from flask_cors import CORS, cross_origin

from tqdm_publisher import TQDMProgressHandler, TQDMProgressPublisher
from tqdm_publisher._demos._parallel_bars._client import (
create_http_server,
find_free_port,
)

N_JOBS = 3

Expand All @@ -35,23 +30,6 @@
progress_handler = TQDMProgressHandler()


def forward_updates_over_server_sent_events(request_id, progress_bar_id, format_dict):
progress_handler.announce(dict(request_id=request_id, progress_bar_id=progress_bar_id, format_dict=format_dict))


class ThreadedHTTPServer:
def __init__(self, port: int, callback):
self.port = port
self.callback = callback

def run(self):
create_http_server(port=self.port, callback=self.callback)

def start(self):
thread = threading.Thread(target=self.run)
thread.start()


def forward_to_http_server(url: str, request_id: str, progress_bar_id: int, format_dict: dict):
"""
This is the parallel callback definition.
Expand Down Expand Up @@ -214,60 +192,50 @@ def listen_to_events():
app = Flask(__name__)
cors = CORS(app)
app.config["CORS_HEADERS"] = "Content-Type"
PORT = find_free_port()
PORT = 3768


@app.route("/start", methods=["POST"])
@cross_origin()
def start():
data = json.loads(request.data) if request.data else {}
request_id = data["request_id"]
run_parallel_processes(all_task_times=TASK_TIMES, request_id=request_id, url=f"http://localhost:{PORT}")
url = f"http://localhost:{PORT}/update"
app.logger.info(url)

run_parallel_processes(all_task_times=TASK_TIMES, request_id=request_id, url=url)
return jsonify({"status": "success"})


@app.route("/events", methods=["GET"])
@app.route("/update", methods=["POST"])
@cross_origin()
def events():
return Response(listen_to_events(), mimetype="text/event-stream")

def update():
data = json.loads(request.data) if request.data else {}
request_id = data["request_id"]
progress_bar_id = data["id"]
format_dict = data["data"]

class ThreadedFlaskServer:
def __init__(self, port: int):
self.port = port
# Forward updates over Sever-Side Events
progress_handler.announce(dict(request_id=request_id, progress_bar_id=progress_bar_id, format_dict=format_dict))

def run(self):
app.run(host="localhost", port=self.port)

def start(self):
thread = threading.Thread(target=self.run)
thread.start()
@app.route("/events", methods=["GET"])
@cross_origin()
def events():
return Response(listen_to_events(), mimetype="text/event-stream")


async def start_server(port):

flask_server = ThreadedFlaskServer(port=3768)
flask_server.start()

def update_queue(request_id: str, progress_bar_id: str, format_dict: dict):
forward_updates_over_server_sent_events(
request_id=request_id, progress_bar_id=progress_bar_id, format_dict=format_dict
)

http_server = ThreadedHTTPServer(port=PORT, callback=update_queue)
http_server.start()

await asyncio.Future()
app.run(host="localhost", port=port)


def run_parallel_bar_demo() -> None:
"""Asynchronously start the servers."""
"""Asynchronously start the server."""
asyncio.run(start_server(port=PORT))


def _run_parallel_bars_demo(port: str, host: str):
URL = f"http://{host}:{port}"

URL = f"http://{host}:{port}/update"
request_id = uuid.uuid4()
run_parallel_processes(all_task_times=TASK_TIMES, request_id=request_id, url=URL)

Expand Down
Loading