From 2551fbae8c6708695a773c30d7374165ab448b2b Mon Sep 17 00:00:00 2001 From: Wei Ouyang Date: Sun, 18 Aug 2024 23:34:45 -0700 Subject: [PATCH 1/3] enhance apps --- hypha/apps.py | 32 +++++++--- hypha/core/__init__.py | 1 + hypha/http.py | 2 +- hypha/plugin_parser.py | 1 + ...-python-app.html => web-python.index.html} | 0 ...-worker-app.html => web-worker.index.html} | 0 .../{window-app.html => window.index.html} | 0 .../hypha-core-app/hypha-app-iframe.html | 22 +++++++ .../hypha-core-app/hypha-app-webpython.js | 19 ++++++ .../hypha-core-app/hypha-app-webworker.js | 1 + hypha/templates/ws/index.html | 62 +++++++++++++++---- 11 files changed, 116 insertions(+), 24 deletions(-) rename hypha/templates/apps/{web-python-app.html => web-python.index.html} (100%) rename hypha/templates/apps/{web-worker-app.html => web-worker.index.html} (100%) rename hypha/templates/apps/{window-app.html => window.index.html} (100%) create mode 100644 hypha/templates/hypha-core-app/hypha-app-iframe.html create mode 100644 hypha/templates/hypha-core-app/hypha-app-webpython.js create mode 100644 hypha/templates/hypha-core-app/hypha-app-webworker.js diff --git a/hypha/apps.py b/hypha/apps.py index 530f91b8..c59218a3 100644 --- a/hypha/apps.py +++ b/hypha/apps.py @@ -131,6 +131,7 @@ async def save_application( app_id: str, card: Card, source: str, + entry_point: str, attachments: Optional[Dict[str, Any]] = None, ): """Save an application to the workspace.""" @@ -159,7 +160,7 @@ async def save_file(key, content): assert "ETag" in response # Upload the source code - await save_file(f"{app_dir}/index.html", source) + await save_file(f"{app_dir}/{entry_point}", source) if attachments: card.attachments = card.attachments or {} @@ -225,7 +226,8 @@ async def install( """Save a server app.""" if template is None: if config: - template = config.get("type") + "-app.html" + config["entry_point"] = config.get("entry_point", "index.html") + template = config.get("type") + "." + config["entry_point"] else: template = "hypha" if not workspace: @@ -265,9 +267,12 @@ async def install( try: config = parse_imjoy_plugin(source) config["source_hash"] = mhash + entry_point = config.get("entry_point", "index.html") + config["entry_point"] = entry_point temp = self.jinja_env.get_template( - safe_join("apps", config["type"] + "-app.html") + safe_join("apps", config["type"] + "." + entry_point) ) + source = temp.render( hypha_main_version=main_version, hypha_rpc_websocket_mjs=self.public_base_url @@ -282,14 +287,18 @@ async def install( f"Failed to parse or compile the hypha app {mhash}: {source[:100]}...", ) from err elif template: + assert "." in template, f"Invalid template name: {template}, should be a file name with extension." + # extract the last dash separated part as the file name temp = self.jinja_env.get_template(safe_join("apps", template)) default_config = { - "name": "Untitled Plugin", + "name": "Untitled App", "version": "0.1.0", "local_base_url": self.local_base_url, } default_config.update(config or {}) config = default_config + entry_point = config.get("entry_point", template) + config["entry_point"] = entry_point source = temp.render( hypha_main_version=main_version, hypha_rpc_websocket_mjs=self.public_base_url @@ -304,17 +313,18 @@ async def install( app_id = f"{mhash}" - public_url = f"{self.public_base_url}/{workspace_info.name}/browser-apps/{app_id}/index.html" + public_url = f"{self.public_base_url}/{workspace_info.name}/server-apps/{app_id}/{entry_point}" card_obj = convert_config_to_card(config, app_id, public_url) card_obj.update( { - "local_url": f"{self.local_base_url}/{workspace_info.name}/browser-apps/{app_id}/index.html", + "local_url": f"{self.local_base_url}/{workspace_info.name}/server-apps/{app_id}/{entry_point}", "public_url": public_url, } ) card = Card.model_validate(card_obj) + assert card.entry_point, "Entry point not found in the card." await self.save_application( - workspace_info.name, app_id, card, source, attachments + workspace_info.name, app_id, card, source, card.entry_point, attachments ) async with self.store.get_workspace_interface( workspace_info.name, user_info @@ -426,10 +436,12 @@ async def start( assert ( app_id in workspace_info.applications ), f"App {app_id} not found in workspace {workspace}, please install it first." - + card = workspace_info.applications[app_id] + entry_point = card.entry_point + assert entry_point, f"Entry point not found for app {app_id}." server_url = self.local_base_url local_url = ( - f"{self.local_base_url}/{workspace}/browser-apps/{app_id}/index.html?" + f"{self.local_base_url}/{workspace}/server-apps/{app_id}/{entry_point}?" + f"client_id={client_id}&workspace={workspace}" + f"&app_id={app_id}" + f"&server_url={server_url}" @@ -439,7 +451,7 @@ async def start( ) server_url = self.public_base_url public_url = ( - f"{self.public_base_url}/{workspace}/browser-apps/{app_id}/index.html?" + f"{self.public_base_url}/{workspace}/server-apps/{app_id}/{entry_point}?" + f"client_id={client_id}&workspace={workspace}" + f"&app_id={app_id}" + f"&server_url={server_url}" diff --git a/hypha/core/__init__.py b/hypha/core/__init__.py index 59d1764c..e4df0cdb 100644 --- a/hypha/core/__init__.py +++ b/hypha/core/__init__.py @@ -239,6 +239,7 @@ class Card(BaseModel): license: Optional[str] = None git_repo: Optional[str] = None source: Optional[str] = None + entry_point: Optional[str] = None services: Optional[List[SerializeAsAny[ServiceInfo]]] = None @classmethod diff --git a/hypha/http.py b/hypha/http.py index c1e8baa1..073f998d 100644 --- a/hypha/http.py +++ b/hypha/http.py @@ -589,7 +589,7 @@ async def send_response(send_queue): except KeyError: return Response(status_code=404) - @router.get(norm_url("/{workspace}/browser-apps/{app_id}/{path:path}")) + @router.get(norm_url("/{workspace}/server-apps/{app_id}/{path:path}")) async def get_browser_app_file( workspace: str, app_id: str, path: str, token: str = None ) -> Response: diff --git a/hypha/plugin_parser.py b/hypha/plugin_parser.py index 7d565929..aad6304a 100644 --- a/hypha/plugin_parser.py +++ b/hypha/plugin_parser.py @@ -122,6 +122,7 @@ def convert_config_to_card(plugin_config, plugin_id, source_url=None): "env", "passive", "services", + "entry_point", ] for field in fields: if field in plugin_config: diff --git a/hypha/templates/apps/web-python-app.html b/hypha/templates/apps/web-python.index.html similarity index 100% rename from hypha/templates/apps/web-python-app.html rename to hypha/templates/apps/web-python.index.html diff --git a/hypha/templates/apps/web-worker-app.html b/hypha/templates/apps/web-worker.index.html similarity index 100% rename from hypha/templates/apps/web-worker-app.html rename to hypha/templates/apps/web-worker.index.html diff --git a/hypha/templates/apps/window-app.html b/hypha/templates/apps/window.index.html similarity index 100% rename from hypha/templates/apps/window-app.html rename to hypha/templates/apps/window.index.html diff --git a/hypha/templates/hypha-core-app/hypha-app-iframe.html b/hypha/templates/hypha-core-app/hypha-app-iframe.html new file mode 100644 index 00000000..55d61188 --- /dev/null +++ b/hypha/templates/hypha-core-app/hypha-app-iframe.html @@ -0,0 +1,22 @@ + + + + Hypha App + + + + + + 
Initializing...
+ + diff --git a/hypha/templates/hypha-core-app/hypha-app-webpython.js b/hypha/templates/hypha-core-app/hypha-app-webpython.js new file mode 100644 index 00000000..d1a52f2e --- /dev/null +++ b/hypha/templates/hypha-core-app/hypha-app-webpython.js @@ -0,0 +1,19 @@ +(()=>{var d=(e,o)=>()=>(o||e((o={exports:{}}).exports,o),o.exports);var a=(e,o,t)=>new Promise((p,c)=>{var y=i=>{try{s(t.next(i))}catch(r){c(r)}},l=i=>{try{s(t.throw(i))}catch(r){c(r)}},s=i=>i.done?p(i.value):Promise.resolve(i.value).then(y,l);s((t=t.apply(e,o)).next())});var m=d(n=>{importScripts("https://cdn.jsdelivr.net/pyodide/v0.26.1/full/pyodide.js");const h=` +import sys +import types +import hypha_rpc +from hypha_rpc import setup_local_client +async def execute(server, config): + print('executing script:', config["name"]) + for script in config["scripts"]: + if script.get("lang") != "python": + raise Exception("Only python scripts are supported") + hypha_rpc.api = server + imjoyModule = types.ModuleType('imjoy_rpc') + imjoyModule.api = server + sys.modules['imjoy'] = imjoyModule + sys.modules['imjoy_rpc'] = imjoyModule + exec(script["content"], {'imjoy': hypha_rpc, 'imjoy_rpc': hypha_rpc, 'hypha_rpc': hypha_rpc, 'api': server}) + +server = await setup_local_client(enable_execution=False, on_ready=execute) +`;console.log("Loading Pyodide...");loadPyodide().then(e=>a(n,null,function*(){console.log("Pyodide is ready to use."),e.setStdout({batched:p=>console.log(p)}),e.setStderr({batched:p=>console.error(p)}),yield e.loadPackage("micropip"),yield e.pyimport("micropip").install("hypha-rpc==0.20.26");const t=typeof window!="undefined";setTimeout(()=>{t?window.parent.postMessage({type:"hyphaClientReady"},"*"):globalThis.postMessage({type:"hyphaClientReady"})},10),yield e.runPythonAsync(h),console.log("Hypha Web Python initialized.")}))});m();})(); diff --git a/hypha/templates/hypha-core-app/hypha-app-webworker.js b/hypha/templates/hypha-core-app/hypha-app-webworker.js new file mode 100644 index 00000000..5fd88df2 --- /dev/null +++ b/hypha/templates/hypha-core-app/hypha-app-webworker.js @@ -0,0 +1 @@ +(()=>{importScripts("https://cdn.jsdelivr.net/npm/hypha-rpc@0.20.26/dist/hypha-rpc-websocket.min.js");hyphaWebsocketClient.setupLocalClient({enable_execution:!0}).then(e=>{console.log("Hypha WebWorker initialized.",e)}).catch(console.error);})(); diff --git a/hypha/templates/ws/index.html b/hypha/templates/ws/index.html index 2d417fca..ed656ed1 100644 --- a/hypha/templates/ws/index.html +++ b/hypha/templates/ws/index.html @@ -7,13 +7,14 @@ Hypha Workspace Management - - - + + + + @@ -73,14 +74,6 @@ margin-left: 60px; } - .collapsible { - display: none; - } - - .collapsed + .collapsible { - display: block; - } - @media (max-width: 768px) { .sidebar { width: 60px; @@ -145,10 +138,41 @@
+ + ``` Use the following code in JavaScript to connect to the server and access an existing service: diff --git a/docs/migration-guide.md b/docs/migration-guide.md index 27ffa911..3cb37bca 100644 --- a/docs/migration-guide.md +++ b/docs/migration-guide.md @@ -15,7 +15,7 @@ To connect to the server, instead of installing the `imjoy-rpc` module, you will pip install -U hypha-rpc # new install ``` -We also changed our versioning strategy, we use the same version number for the server and client, so it's easier to match the client and server versions. For example, `hypha-rpc` version `0.20.26` is compatible with Hypha server version `0.20.26`. +We also changed our versioning strategy, we use the same version number for the server and client, so it's easier to match the client and server versions. For example, `hypha-rpc` version `0.20.27` is compatible with Hypha server version `0.20.27`. #### 2. Change the imports to use `hypha-rpc` @@ -128,10 +128,10 @@ loop.run_forever() To connect to the server, instead of using the `imjoy-rpc` module, you will need to use the `hypha-rpc` module. The `hypha-rpc` module is a standalone module that provides the RPC connection to the Hypha server. You can include it in your HTML using a script tag: ```html - + ``` -We also changed our versioning strategy, we use the same version number for the server and client, so it's easier to match the client and server versions. For example, `hypha-rpc` version `0.20.26` is compatible with Hypha server version `0.20.26`. +We also changed our versioning strategy, we use the same version number for the server and client, so it's easier to match the client and server versions. For example, `hypha-rpc` version `0.20.27` is compatible with Hypha server version `0.20.27`. #### 2. Change the connection method and use camelCase for service function names @@ -149,7 +149,7 @@ Here is a suggested list of search and replace operations to update your code: Here is an example of how the updated code might look: ```html - + + + + diff --git a/hypha/templates/ws/index.html b/hypha/templates/ws/index.html index ed656ed1..0ca84244 100644 --- a/hypha/templates/ws/index.html +++ b/hypha/templates/ws/index.html @@ -139,8 +139,18 @@
@@ -336,7 +356,7 @@

Clients

{panel === 'development' && (

Development panel with Hypha server simulation

-
@@ -365,7 +385,9 @@

Clients

React.useEffect(() => { // Connect to the Hypha server const connectToServer = async () => { - setWs(await hyphaWebsocketClient.connectToServer(config)); + const remoteServer = await hyphaWebsocketClient.connectToServer(config); + window.remoteServer = remoteServer; + setWs(remoteServer); }; connectToServer(); }, []); diff --git a/hypha/websocket.py b/hypha/websocket.py index 4d5a4734..535b15ff 100644 --- a/hypha/websocket.py +++ b/hypha/websocket.py @@ -153,7 +153,7 @@ async def websocket_endpoint( workspace, client_id, user_info, - status.status.WS_1001_GOING_AWAY, + status.WS_1001_GOING_AWAY, e, ) diff --git a/requirements.txt b/requirements.txt index 3e50631a..7581e661 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ aioboto3==11.2.0 aiofiles==23.2.1 base58==2.1.1 fastapi==0.106.0 -hypha-rpc==0.20.26 +hypha-rpc==0.20.27 jinja2==3.1.2 lxml==4.9.3 msgpack==1.0.5 diff --git a/setup.py b/setup.py index 3c46750c..631d49e3 100644 --- a/setup.py +++ b/setup.py @@ -12,7 +12,7 @@ REQUIREMENTS = [ "aiofiles", "fastapi>=0.70.0,<=0.106.0", - "hypha-rpc>=0.20.26", + "hypha-rpc>=0.20.27", "msgpack>=1.0.2", "numpy", "pydantic[email]>=2.6.1", diff --git a/tests/conftest.py b/tests/conftest.py index 6f7637c4..bc1e521b 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -60,7 +60,7 @@ def _generate_token(id, roles): scope=create_scope(workspaces={f"ws-user-{id}": UserPermission.admin}), expires_at=None, ) - token = generate_presigned_token(root_user_info) + token = generate_presigned_token(root_user_info, 18000) yield token @@ -77,7 +77,7 @@ def generate_root_user_token(): scope=create_scope(workspaces={"*": UserPermission.admin}), expires_at=None, ) - token = generate_presigned_token(root_user_info) + token = generate_presigned_token(root_user_info, 1800) yield token From 2f9355baa139151fb2352794e58c19267c773983 Mon Sep 17 00:00:00 2001 From: Wei Ouyang Date: Mon, 19 Aug 2024 23:36:14 -0700 Subject: [PATCH 3/3] Fix base url --- tests/conftest.py | 37 +++++++++++++++++++++++++++++++++++-- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index bc1e521b..3a7ba16a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -16,6 +16,7 @@ from hypha.core import UserInfo, auth, UserPermission from hypha.core.auth import generate_presigned_token, create_scope from hypha.minio import setup_minio_executables +from redis import Redis from . import ( MINIO_PORT, @@ -121,6 +122,34 @@ def triton_server(): yield +@pytest_asyncio.fixture(name="redis_server", scope="session") +def redis_server(): + """Start a redis server as test fixture and tear down after test.""" + try: + r = Redis(host="localhost", port=REDIS_PORT) + r.ping() + yield + except Exception: + # user docker to start redis + subprocess.Popen( + ["docker", "run", "-d", "-p", f"{REDIS_PORT}:6379", "redis:6.2.5"] + ) + timeout = 10 + while timeout > 0: + try: + r = Redis(host="localhost", port=REDIS_PORT) + r.ping() + break + except Exception: + pass + timeout -= 0.1 + time.sleep(0.1) + yield + subprocess.Popen(["docker", "stop", "redis"]) + subprocess.Popen(["docker", "rm", "redis"]) + time.sleep(1) + + @pytest_asyncio.fixture(name="fastapi_server", scope="session") def fastapi_server_fixture(minio_server): """Start server as test fixture and tear down after test.""" @@ -166,7 +195,7 @@ def fastapi_server_fixture(minio_server): @pytest_asyncio.fixture(name="fastapi_server_redis_1", scope="session") -def fastapi_server_redis_1(minio_server): +def fastapi_server_redis_1(redis_server, minio_server): """Start server as test fixture and tear down after test.""" with subprocess.Popen( [ @@ -176,6 +205,8 @@ def fastapi_server_redis_1(minio_server): f"--port={SIO_PORT_REDIS_1}", # "--enable-server-apps", # "--enable-s3", + # need to define it so the two server can communicate + f"--public-base-url=http://my-public-url.com", f"--redis-uri=redis://127.0.0.1:{REDIS_PORT}/0", "--reset-redis", # f"--endpoint-url={MINIO_SERVER_URL}", @@ -205,7 +236,7 @@ def fastapi_server_redis_1(minio_server): @pytest_asyncio.fixture(name="fastapi_server_redis_2", scope="session") -def fastapi_server_redis_2(minio_server, fastapi_server): +def fastapi_server_redis_2(redis_server, minio_server, fastapi_server): """Start a backup server as test fixture and tear down after test.""" with subprocess.Popen( [ @@ -215,6 +246,8 @@ def fastapi_server_redis_2(minio_server, fastapi_server): f"--port={SIO_PORT_REDIS_2}", # "--enable-server-apps", # "--enable-s3", + # need to define it so the two server can communicate + f"--public-base-url=http://my-public-url.com", f"--redis-uri=redis://127.0.0.1:{REDIS_PORT}/0", # f"--endpoint-url={MINIO_SERVER_URL}", # f"--access-key-id={MINIO_ROOT_USER}",