From ff975c6ce7133c6ee83e9e1a93a2db1dd166d02f Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Sun, 28 May 2023 18:56:41 -0400 Subject: [PATCH 01/54] typo I think --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7ea5c7e8..37bc1291 100644 --- a/setup.py +++ b/setup.py @@ -99,7 +99,7 @@ 'Topic :: Scientific/Engineering :: Medical Science Apps.', 'Intended Audience :: Science/Research' ], - extras_requires = { + extras_require = { 'sbml': [ 'python_libsbml', 'lxml', From 07143f8eb1d93c400d9dde926f55ef05b8424eb6 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Sun, 28 May 2023 18:57:13 -0400 Subject: [PATCH 02/54] mass import --- gillespy2/remote/__init__.py | 24 + gillespy2/remote/__version__.py | 38 + gillespy2/remote/client/__init__.py | 20 + gillespy2/remote/client/compute_server.py | 46 ++ gillespy2/remote/client/endpoint.py | 28 + gillespy2/remote/client/server.py | 120 ++++ gillespy2/remote/cloud/__init__.py | 21 + gillespy2/remote/cloud/ec2.py | 651 ++++++++++++++++++ gillespy2/remote/cloud/ec2_config.py | 119 ++++ gillespy2/remote/cloud/exceptions.py | 41 ++ gillespy2/remote/core/__init__.py | 22 + gillespy2/remote/core/errors.py | 29 + gillespy2/remote/core/exceptions.py | 24 + gillespy2/remote/core/log_config.py | 36 + gillespy2/remote/core/messages/__init__.py | 0 gillespy2/remote/core/messages/base.py | 52 ++ gillespy2/remote/core/messages/results.py | 72 ++ .../remote/core/messages/simulation_run.py | 119 ++++ .../core/messages/simulation_run_unique.py | 89 +++ gillespy2/remote/core/messages/source_ip.py | 68 ++ gillespy2/remote/core/messages/status.py | 107 +++ gillespy2/remote/core/remote_results.py | 138 ++++ gillespy2/remote/core/remote_simulation.py | 181 +++++ gillespy2/remote/launch.py | 152 ++++ gillespy2/remote/server/__init__.py | 20 + gillespy2/remote/server/api.py | 107 +++ gillespy2/remote/server/cache.py | 173 +++++ gillespy2/remote/server/is_cached.py | 87 +++ gillespy2/remote/server/results.py | 70 ++ gillespy2/remote/server/results_unique.py | 72 ++ gillespy2/remote/server/run.py | 116 ++++ gillespy2/remote/server/run_unique.py | 127 ++++ gillespy2/remote/server/sourceip.py | 46 ++ gillespy2/remote/server/status.py | 157 +++++ setup.py | 15 + 35 files changed, 3187 insertions(+) create mode 100644 gillespy2/remote/__init__.py create mode 100644 gillespy2/remote/__version__.py create mode 100644 gillespy2/remote/client/__init__.py create mode 100644 gillespy2/remote/client/compute_server.py create mode 100644 gillespy2/remote/client/endpoint.py create mode 100644 gillespy2/remote/client/server.py create mode 100644 gillespy2/remote/cloud/__init__.py create mode 100644 gillespy2/remote/cloud/ec2.py create mode 100644 gillespy2/remote/cloud/ec2_config.py create mode 100644 gillespy2/remote/cloud/exceptions.py create mode 100644 gillespy2/remote/core/__init__.py create mode 100644 gillespy2/remote/core/errors.py create mode 100644 gillespy2/remote/core/exceptions.py create mode 100644 gillespy2/remote/core/log_config.py create mode 100644 gillespy2/remote/core/messages/__init__.py create mode 100644 gillespy2/remote/core/messages/base.py create mode 100644 gillespy2/remote/core/messages/results.py create mode 100644 gillespy2/remote/core/messages/simulation_run.py create mode 100644 gillespy2/remote/core/messages/simulation_run_unique.py create mode 100644 gillespy2/remote/core/messages/source_ip.py create mode 100644 gillespy2/remote/core/messages/status.py create mode 100644 gillespy2/remote/core/remote_results.py create mode 100644 gillespy2/remote/core/remote_simulation.py create mode 100644 gillespy2/remote/launch.py create mode 100644 gillespy2/remote/server/__init__.py create mode 100644 gillespy2/remote/server/api.py create mode 100644 gillespy2/remote/server/cache.py create mode 100644 gillespy2/remote/server/is_cached.py create mode 100644 gillespy2/remote/server/results.py create mode 100644 gillespy2/remote/server/results_unique.py create mode 100644 gillespy2/remote/server/run.py create mode 100644 gillespy2/remote/server/run_unique.py create mode 100644 gillespy2/remote/server/sourceip.py create mode 100644 gillespy2/remote/server/status.py diff --git a/gillespy2/remote/__init__.py b/gillespy2/remote/__init__.py new file mode 100644 index 00000000..5317a622 --- /dev/null +++ b/gillespy2/remote/__init__.py @@ -0,0 +1,24 @@ +''' +stochss_compute +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from stochss_compute.client import * +from stochss_compute.core import * +from stochss_compute.server import * +from stochss_compute.cloud import * \ No newline at end of file diff --git a/gillespy2/remote/__version__.py b/gillespy2/remote/__version__.py new file mode 100644 index 00000000..49f22c08 --- /dev/null +++ b/gillespy2/remote/__version__.py @@ -0,0 +1,38 @@ +''' +stochss_compute.__version__ +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +# ============================================================================= +# @file __version__.py +# @brief stochss-compute version info +# @license Please see the file named LICENSE.md in parent directory +# @website https://github.com/StochSS/stochss-compute +# ============================================================================= + + +__version__ = '1.0.1' + +__title__ = "stochss-compute" +__description__ = "A compute delegation package for the StochSS family of stochastic simulators" +__url__ = "https://github.com/StochSS/stochss-compute" +__download_url__ = "https://pypi.org/project/stochss-compute/#files" +__author__ = "See CONTRIBUTORS.md" +__email__ = "briandrawert@gmail.com" +__license__ = "GPLv3" +__copyright__ = "Copyright (C) 2017-2023" diff --git a/gillespy2/remote/client/__init__.py b/gillespy2/remote/client/__init__.py new file mode 100644 index 00000000..1c53bef9 --- /dev/null +++ b/gillespy2/remote/client/__init__.py @@ -0,0 +1,20 @@ +''' +stochss_compute.client +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .compute_server import ComputeServer diff --git a/gillespy2/remote/client/compute_server.py b/gillespy2/remote/client/compute_server.py new file mode 100644 index 00000000..9d984f21 --- /dev/null +++ b/gillespy2/remote/client/compute_server.py @@ -0,0 +1,46 @@ +''' +ComputeServer(Server) +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from stochss_compute.client.server import Server + +class ComputeServer(Server): + ''' + Simple object representing a remote instance of StochSS-Compute. + + :param host: Address of the remote server. + :type host: str + + :param port: Port on which to connect. Defaults to 29681. + :type port: int + ''' + # pylint: disable=super-init-not-called + def __init__(self, host: str, port: int = 29681): + host = host.replace('http://','') + host = host.split(':')[0] + self._address = f"http://{host}:{port}" + # pylint: enable=super-init-not-called + + @property + def address(self) -> str: + """ + The server's IP address and port. + + :returns: "http://{host}:{port}" + """ + return self._address diff --git a/gillespy2/remote/client/endpoint.py b/gillespy2/remote/client/endpoint.py new file mode 100644 index 00000000..680a780a --- /dev/null +++ b/gillespy2/remote/client/endpoint.py @@ -0,0 +1,28 @@ +''' +Endpoint(Enum) +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from enum import Enum + +class Endpoint(Enum): + ''' + API Endpoints. + ''' + SIMULATION_GILLESPY2 = 1 + CLOUD = 2 + \ No newline at end of file diff --git a/gillespy2/remote/client/server.py b/gillespy2/remote/client/server.py new file mode 100644 index 00000000..ae7b6bb5 --- /dev/null +++ b/gillespy2/remote/client/server.py @@ -0,0 +1,120 @@ +''' +Server(ABC) +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from time import sleep +from abc import ABC, abstractmethod +import requests +from stochss_compute.client.endpoint import Endpoint +from stochss_compute.core.messages.base import Request + +class Server(ABC): + ''' + Abstract Server class with hard coded endpoints. + + :raises TypeError: Server cannot be instantiated directly. Must be ComputeServer or Cluster. + ''' + + _endpoints = { + Endpoint.SIMULATION_GILLESPY2: "/api/v2/simulation/gillespy2", + Endpoint.CLOUD: "/api/v2/cloud" + } + + def __init__(self) -> None: + raise TypeError('Server cannot be instantiated directly. Must be ComputeServer or Cluster.') + + @property + @abstractmethod + def address(self): + ''' + NotImplemented + ''' + return NotImplemented + + def get(self, endpoint: Endpoint, sub: str): + ''' + Send a GET request to endpoint. + + :param endpoint: The API endpoint. + :type endpoint: Endpoint + + :param sub: Final part of url string. + :type sub: str + + :returns: The HTTP response. + :rtype: requests.Response + ''' + url = f"{self.address}{self._endpoints[endpoint]}{sub}" + n_try = 1 + sec = 3 + while n_try <= 3: + try: + return requests.get(url, timeout=30) + + except ConnectionError: + print(f"Connection refused by server. Retrying in {sec} seconds....") + sleep(sec) + n_try += 1 + sec *= n_try + except Exception as err: + print(f"Unknown error: {err}. Retrying in {sec} seconds....") + sleep(sec) + n_try += 1 + sec *= n_try + + def post(self, endpoint: Endpoint, sub: str, request: Request = None): + ''' + Send a POST request to endpoint. + + :param endpoint: The API endpoint. + :type endpoint: Endpoint + + :param sub: Final part of url string. + :type sub: str + + :param request: An object that inherits from Request. + :type request: Request + + :returns: The HTTP response. + :rtype: requests.Response + ''' + + if self.address is NotImplemented: + raise NotImplementedError + + url = f"{self.address}{self._endpoints[endpoint]}{sub}" + n_try = 1 + sec = 3 + while n_try <= 3: + try: + if request is None: + print(f"[POST] {url}") + return requests.post(url, timeout=30) + print(f"[{type(request).__name__}] {url}") + return requests.post(url, json=request.encode(), timeout=30) + + except ConnectionError: + print(f"Connection refused by server. Retrying in {sec} seconds....") + sleep(sec) + n_try += 1 + sec *= n_try + except Exception as err: + print(f"Unknown error: {err}. Retrying in {sec} seconds....") + sleep(sec) + n_try += 1 + sec *= n_try diff --git a/gillespy2/remote/cloud/__init__.py b/gillespy2/remote/cloud/__init__.py new file mode 100644 index 00000000..eb22eb6e --- /dev/null +++ b/gillespy2/remote/cloud/__init__.py @@ -0,0 +1,21 @@ +''' +stochss_compute.cloud +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .ec2 import EC2Cluster +from .ec2_config import EC2LocalConfig, EC2RemoteConfig diff --git a/gillespy2/remote/cloud/ec2.py b/gillespy2/remote/cloud/ec2.py new file mode 100644 index 00000000..5d458513 --- /dev/null +++ b/gillespy2/remote/cloud/ec2.py @@ -0,0 +1,651 @@ +''' +stochss_compute.cloud.ec2 +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import logging +from time import sleep +from secrets import token_hex +from stochss_compute.client.server import Server +from stochss_compute.cloud.ec2_config import EC2LocalConfig, EC2RemoteConfig +from stochss_compute.core.messages.source_ip import SourceIpRequest, SourceIpResponse +from stochss_compute.cloud.exceptions import EC2ImportException, ResourceException, EC2Exception +from stochss_compute.client.endpoint import Endpoint +try: + import boto3 + from botocore.config import Config + from botocore.session import get_session + from botocore.exceptions import ClientError + from paramiko import SSHClient, AutoAddPolicy +except ImportError as err: + raise EC2ImportException from err + + +def _ec2_logger(): + log = logging.getLogger("EC2Cluster") + log.setLevel(logging.INFO) + log.propagate = False + + if not log.handlers: + _formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + _handler = logging.StreamHandler() + _handler.setFormatter(_formatter) + log.addHandler(_handler) + + return log + + +class EC2Cluster(Server): + """ + Attempts to load a StochSS-Compute cluster. Otherwise just initializes a new cluster. + + :param local_config: Optional. Allows configuration of local cluster resources. + :type local_config: EC2LocalConfig + + :param remote_config: Optional. Allows configuration of remote cluster resource identifiers. + :type remote_config: EC2RemoteConfig + + :raises EC2Exception: possible boto3 ClientError from AWS calls. See `here `_. + """ + log = _ec2_logger() + + _init = False + _client = None + _resources = None + _restricted: bool = False + _subnets = { + 'public': None, + 'private': None + } + _default_security_group = None + _server_security_group = None + _vpc = None + _server = None + _ami = None + + _local_config = EC2LocalConfig() + _remote_config = EC2RemoteConfig() + + def __init__(self, local_config=None, remote_config=None) -> None: + if local_config is not None: + self._local_config = local_config + if remote_config is not None: + self._remote_config = remote_config + + if self._remote_config.region is not None: + config = Config(region_name=self._remote_config.region) + region = self._remote_config.region + # Overrides any underlying configurationz + self._client = boto3.client('ec2', config=config) + self._resources = boto3.resource('ec2', config=config) + else: + region = get_session().get_config_variable('region') + self._client = boto3.client('ec2') + self._resources = boto3.resource('ec2') + + if self._remote_config.ami is not None: + self._ami = self._remote_config.ami + else: + try: + self._ami = self._remote_config._AMIS[region] + except KeyError as err2: + self._set_status('region error') + raise EC2Exception(f'Unsupported region. Currently Supported: \ + {list(self._remote_config._AMIS.keys())}. \ + Try providing an AMI identifier.') from err2 + + try: + self._load_cluster() + except ClientError as c_e: + self._set_status(c_e.response['Error']['Code']) + raise EC2Exception(c_e.response['Error']['Message']) from c_e + except ResourceException: + self.clean_up() + + @property + def address(self) -> str: + """ + The server's IP address and port. + + :returns: "http://{ip}:{port}" + :rtype: str + + :raises EC2Exception: Do not call before launching a cluster. + """ + if self._server is None: + raise EC2Exception('No server found. First launch a cluster.') + if self._server.public_ip_address is None: + self._server.reload() + if self._server.public_ip_address is None: + raise EC2Exception('No public address found.') + + return f'http://{self._server.public_ip_address}:{self._remote_config.api_port}' + + @property + def status(self) -> str: + ''' + Return the EC2 instance status. + + :returns: A status set locally, or, if connected, a status fetched from the instance. + :rtype: str + ''' + if self._server is None: + return self._status + else: + return self._server.state['Name'] + + def _set_status(self, status): + self._status = status + if self._local_config.status_file is not None: + with open(self._local_config.status_file, 'w', encoding='utf-8') as file: + file.write(status) + + def launch_single_node_instance(self, instance_type): + """ + Launches a single node StochSS-Compute instance. Make sure to check instance_type pricing before launching. + + :param instance_type: Example: 't3.nano' See full list `here `_. + :type instance_type: str + + :raises EC2Exception: possible boto3 ClientError from AWS calls. See `here `_. + """ + if self._init is True: + raise EC2Exception('You cannot launch more than one \ + StochSS-Compute cluster instance \ + per EC2Cluster object.') + + self._set_status('launching') + try: + self._launch_network() + self._create_root_key() + self._launch_head_node(instance_type=instance_type) + except ClientError as c_e: + self._set_status(c_e.response['Error']['Code']) + raise EC2Exception(c_e.response['Error']['Message']) from c_e + self._set_status(self._server.state['Name']) + + def clean_up(self): + """ + Terminates and removes all cluster resources. + + :raises EC2Exception: possible boto3 ClientError from AWS calls. See `here `_. + """ + self._set_status('terminating') + self._init = False + + vpc_search_filter = [ + { + 'Name': 'tag:Name', + 'Values': [ + self._remote_config.vpc_name + ] + } + ] + try: + vpc_response = self._client.describe_vpcs( + Filters=vpc_search_filter) + for vpc_dict in vpc_response['Vpcs']: + vpc_id = vpc_dict['VpcId'] + vpc = self._resources.Vpc(vpc_id) + for instance in vpc.instances.all(): + instance.terminate() + self.log.info( + 'Terminating "%s". This might take a minute.......', instance.id) + instance.wait_until_terminated() + self._server = None + self.log.info('Instance "%s" terminated.', instance.id) + for s_g in vpc.security_groups.all(): + if s_g.group_name == self._remote_config.security_group_name: + self.log.info('Deleting "%s".......', s_g.id) + s_g.delete() + self._server_security_group = None + self.log.info('Security group "%s" deleted.', s_g.id) + elif s_g.group_name == 'default': + self._default_security_group = None + for subnet in vpc.subnets.all(): + self.log.info('Deleting %s.......', subnet.id) + subnet.delete() + self._subnets['public'] = None + self.log.info('Subnet %s deleted.', subnet.id) + for igw in vpc.internet_gateways.all(): + self.log.info('Detaching %s.......', igw.id) + igw.detach_from_vpc(VpcId=vpc.vpc_id) + self.log.info('Gateway %s detached.', igw.id) + self.log.info('Deleting %s.......', igw.id) + igw.delete() + self.log.info('Gateway %s deleted.', igw.id) + self.log.info('Deleting %s.......', vpc.id) + vpc.delete() + self._vpc = None + self.log.info('VPC %s deleted.', vpc.id) + try: + self._client.describe_key_pairs( + KeyNames=[self._remote_config.key_name]) + key_pair = self._resources.KeyPair( + self._remote_config.key_name) + self.log.info( + 'Deleting "%s".', self._remote_config.key_name) + self.log.info( + 'Key "%s" deleted.', self._remote_config.key_name) + key_pair.delete() + except: + pass + except ClientError as c_e: + self._set_status(c_e.response['Error']['Code']) + raise EC2Exception(c_e.response['Error']['Message']) from c_e + self._delete_root_key() + self._set_status('terminated') + + def _launch_network(self): + """ + Launches required network resources. + """ + self.log.info("Launching Network.......") + self._create_sssc_vpc() + self._create_sssc_subnet(public=True) + self._create_sssc_subnet(public=False) + self._create_sssc_security_group() + self._vpc.reload() + + def _create_root_key(self): + """ + Creates a key pair for SSH login and instance launch. + """ + + response = self._client.create_key_pair( + KeyName=self._remote_config.key_name, + KeyType=self._local_config.key_type, + KeyFormat=self._local_config.key_format) + + waiter = self._client.get_waiter('key_pair_exists') + waiter.wait(KeyNames=[self._remote_config.key_name]) + os.makedirs(self._local_config.key_dir, exist_ok=True) + with open(self._local_config.key_path, 'x', encoding='utf-8') as key: + key.write(response['KeyMaterial']) + os.chmod(self._local_config.key_path, 0o400) + + def _delete_root_key(self) -> None: + """ + Deletes key from local filesystem if it exists. + """ + if os.path.exists(self._local_config.key_path): + self.log.info( + 'Deleting "%s".', self._local_config.key_path) + os.remove(self._local_config.key_path) + self.log.info('"%s" deleted.', self._local_config.key_path) + + def _create_sssc_vpc(self): + """ + Creates a vpc. + """ + vpc_cidr_block = '172.31.0.0/16' + vpc_tag = [ + { + 'ResourceType': 'vpc', + 'Tags': [ + { + 'Key': 'Name', + 'Value': self._remote_config.vpc_name + } + ] + } + ] + + vpc_response = self._client.create_vpc( + CidrBlock=vpc_cidr_block, TagSpecifications=vpc_tag) + vpc_id = vpc_response['Vpc']['VpcId'] + vpc_waiter_exist = self._client.get_waiter('vpc_exists') + vpc_waiter_exist.wait(VpcIds=[vpc_id]) + vpc_waiter_avail = self._client.get_waiter('vpc_available') + vpc_waiter_avail.wait(VpcIds=[vpc_id]) + self._vpc = self._resources.Vpc(vpc_id) + self._default_security_group = list( + sg for sg in self._vpc.security_groups.all())[0] + + self._client.modify_vpc_attribute( + VpcId=vpc_id, EnableDnsSupport={'Value': True}) + self._client.modify_vpc_attribute( + VpcId=vpc_id, EnableDnsHostnames={'Value': True}) + + igw_response = self._client.create_internet_gateway() + igw_id = igw_response['InternetGateway']['InternetGatewayId'] + igw_waiter = self._client.get_waiter('internet_gateway_exists') + igw_waiter.wait(InternetGatewayIds=[igw_id]) + + self._vpc.attach_internet_gateway(InternetGatewayId=igw_id) + for rtb in self._vpc.route_tables.all(): + if rtb.associations_attribute[0]['Main'] is True: + rtb_id = rtb.route_table_id + self._client.create_route( + RouteTableId=rtb_id, GatewayId=igw_id, DestinationCidrBlock='0.0.0.0/0') + + self._vpc.reload() + + def _create_sssc_subnet(self, public: bool): + """ + Creates a public or private subnet. + """ + if public is True: + label = 'public' + subnet_cidr_block = '172.31.0.0/20' + else: + label = 'private' + subnet_cidr_block = '172.31.16.0/20' + + subnet_tag = [ + { + 'ResourceType': 'subnet', + 'Tags': [ + { + 'Key': 'Name', + 'Value': f'{self._remote_config.subnet_name}-{label}' + } + ] + } + ] + self._subnets[label] = self._vpc.create_subnet( + CidrBlock=subnet_cidr_block, TagSpecifications=subnet_tag) + waiter = self._client.get_waiter('subnet_available') + waiter.wait(SubnetIds=[self._subnets[label].id]) + self._client.modify_subnet_attribute( + SubnetId=self._subnets[label].id, MapPublicIpOnLaunch={'Value': True}) + self._subnets[label].reload() + + def _create_sssc_security_group(self): + """ + Creates a security group for SSH and StochSS-Compute API access. + """ + description = 'Default Security Group for StochSS-Compute.' + self._server_security_group = self._vpc.create_security_group( + Description=description, GroupName=self._remote_config.security_group_name) + sshargs = { + 'CidrIp': '0.0.0.0/0', + 'FromPort': 22, + 'ToPort': 22, + 'IpProtocol': 'tcp', + } + self._server_security_group.authorize_ingress(**sshargs) + sgargs = { + 'CidrIp': '0.0.0.0/0', + 'FromPort': self._remote_config.api_port, + 'ToPort': self._remote_config.api_port, + 'IpProtocol': 'tcp', + 'TagSpecifications': [ + { + 'ResourceType': 'security-group-rule', + 'Tags': [ + { + 'Key': 'Name', + 'Value': 'api-server' + }, + ] + }, + ] + } + self._server_security_group.authorize_ingress(**sgargs) + self._server_security_group.reload() + + def _restrict_ingress(self, ip_address: str = ''): + """ + Modifies the security group API ingress rule to + only allow access on the specified port from the given ip address. + """ + rule_filter = [ + { + 'Name': 'group-id', + 'Values': [ + self._server_security_group.id, + ] + }, + { + 'Name': 'tag:Name', + 'Values': [ + 'api-server', + ] + }, + ] + sgr_response = self._client.describe_security_group_rules( + Filters=rule_filter) + sgr_id = sgr_response['SecurityGroupRules'][0]['SecurityGroupRuleId'] + new_sg_rules = [ + { + 'SecurityGroupRuleId': sgr_id, + 'SecurityGroupRule': { + 'IpProtocol': 'tcp', + 'FromPort': self._remote_config.api_port, + 'ToPort': self._remote_config.api_port, + 'CidrIpv4': f'{ip_address}/32', + 'Description': 'Restricts cluster access.' + } + }, + ] + self._client.modify_security_group_rules( + GroupId=self._server_security_group.id, SecurityGroupRules=new_sg_rules) + self._server_security_group.reload() + + def _launch_head_node(self, instance_type): + """ + Launches a StochSS-Compute server instance. + """ + cloud_key = token_hex(32) + + launch_commands = f'''#!/bin/bash +sudo yum update -y +sudo yum -y install docker +sudo usermod -a -G docker ec2-user +sudo service docker start +sudo chmod 666 /var/run/docker.sock +docker run --network host --rm -t -e CLOUD_LOCK={cloud_key} --name sssc stochss/stochss-compute:cloud stochss-compute-cluster -p {self._remote_config.api_port} > /home/ec2-user/sssc-out 2> /home/ec2-user/sssc-err & +''' + kwargs = { + 'ImageId': self._ami, + 'InstanceType': instance_type, + 'KeyName': self._remote_config.key_name, + 'MinCount': 1, + 'MaxCount': 1, + 'SubnetId': self._subnets['public'].id, + 'SecurityGroupIds': [self._default_security_group.id, self._server_security_group.id], + 'TagSpecifications': [ + { + 'ResourceType': 'instance', + 'Tags': [ + { + 'Key': 'Name', + 'Value': self._remote_config.server_name + }, + ] + }, + ], + 'UserData': launch_commands, + } + + self.log.info( + 'Launching StochSS-Compute server instance. This might take a minute.......') + try: + response = self._client.run_instances(**kwargs) + except ClientError as c_e: + raise EC2Exception from c_e + + instance_id = response['Instances'][0]['InstanceId'] + # try catch + self._server = self._resources.Instance(instance_id) + self._server.wait_until_exists() + self._server.wait_until_running() + + self.log.info('Instance "%s" is running.', instance_id) + + self._poll_launch_progress(['sssc']) + + self.log.info('Restricting server access to only your ip.') + source_ip = self._get_source_ip(cloud_key) + + self._restrict_ingress(source_ip) + self._init = True + self.log.info('StochSS-Compute ready to go!') + + def _poll_launch_progress(self, container_names, mock=False): + """ + Polls the instance to see if the Docker container is running. + + :param container_names: A list of Docker container names to check against. + :type container_names: List[str] + """ + if mock is True: + from test.unit_tests.mock_ssh import MockSSH + ssh = MockSSH() + else: + ssh = SSHClient() + ssh.set_missing_host_key_policy(AutoAddPolicy()) + sshtries = 0 + while True: + try: + ssh.connect(self._server.public_ip_address, username='ec2-user', + key_filename=self._local_config.key_path, look_for_keys=False) + break + except Exception as err2: + if sshtries >= 5: + raise err2 + self._server.reload() + sleep(5) + sshtries += 1 + continue + for container in container_names: + sshtries = 0 + while True: + sleep(60) + _, stdout, stderr = ssh.exec_command( + "docker container inspect -f '{{.State.Running}}' " + f'{container}') + rc = stdout.channel.recv_exit_status() + out = stdout.readlines() + err2 = stderr.readlines() + if rc == -1: + ssh.close() + raise EC2Exception( + "Something went wrong connecting to the server. No exit status provided by the server.") + # Wait for yum update, docker install, container download + if rc == 1 or rc == 127: + self.log.info('Waiting on Docker daemon.') + sshtries += 1 + if sshtries >= 5: + ssh.close() + raise EC2Exception( + f"Something went wrong with Docker. Max retry attempts exceeded.\nError:\n{''.join(err2)}") + if rc == 0: + if 'true\n' in out: + sleep(10) + self.log.info('Container "%s" is running.', container) + break + ssh.close() + + def _get_source_ip(self, cloud_key): + """ + Ping the server to find the IP address associated with the request. + + :param cloud_key: A secret key which must match the random key used to launch the instance. + :type cloud_key: str + """ + source_ip_request = SourceIpRequest(cloud_key=cloud_key) + response_raw = self.post( + Endpoint.CLOUD, sub='/sourceip', request=source_ip_request) + if not response_raw.ok: + raise EC2Exception(response_raw.reason) + response = SourceIpResponse.parse(response_raw.text) + return response.source_ip + + def _load_cluster(self): + ''' + Reload cluster resources. Returns False if no vpc named sssc-vpc. + ''' + + vpc_search_filter = [ + { + 'Name': 'tag:Name', + 'Values': [ + self._remote_config.vpc_name + ] + } + ] + vpc_response = self._client.describe_vpcs(Filters=vpc_search_filter) + + if len(vpc_response['Vpcs']) == 0: + if os.path.exists(self._local_config.key_path): + self._set_status('key error') + raise ResourceException + else: + try: + keypair = self._client.describe_key_pairs( + KeyNames=[self._remote_config.key_name]) + if keypair is not None: + self._set_status('key error') + raise ResourceException + except: + pass + return False + if len(vpc_response['Vpcs']) == 2: + self.log.error('More than one VPC named "%s".', + self._remote_config.vpc_name) + self._set_status('VPC error') + raise ResourceException + vpc_id = vpc_response['Vpcs'][0]['VpcId'] + self._vpc = self._resources.Vpc(vpc_id) + vpc = self._vpc + errors = False + for instance in vpc.instances.all(): + for tag in instance.tags: + if tag['Key'] == 'Name' and tag['Value'] == self._remote_config.server_name: + self._server = instance + if self._server is None: + self.log.warn('No instances named "%s".', + self._remote_config.server_name) + self._set_status('server error') + errors = True + for s_g in vpc.security_groups.all(): + if s_g.group_name == 'default': + self._default_security_group = s_g + if s_g.group_name == self._remote_config.security_group_name: + for rule in s_g.ip_permissions: + if rule['FromPort'] == self._remote_config.api_port \ + and rule['ToPort'] == self._remote_config.api_port \ + and rule['IpRanges'][0]['CidrIp'] == '0.0.0.0/0': + self.log.warn('Security Group rule error.') + self._set_status('security group error') + errors = True + self._server_security_group = s_g + if self._server_security_group is None: + self.log.warn('No security group named "%s".', + self._remote_config.security_group_name) + self._set_status('security group error') + errors = True + for subnet in vpc.subnets.all(): + for tag in subnet.tags: + if tag['Key'] == 'Name' and tag['Value'] == f'{self._remote_config.subnet_name}-public': + self._subnets['public'] = subnet + if tag['Key'] == 'Name' and tag['Value'] == f'{self._remote_config.subnet_name}-private': + self._subnets['private'] = subnet + if None in self._subnets.values(): + self.log.warn('Missing or misconfigured subnet.') + self._set_status('subnet error') + errors = True + if errors is True: + raise ResourceException + else: + self._init = True + self.log.info('Cluster loaded.') + self._set_status(self._server.state['Name']) + return True diff --git a/gillespy2/remote/cloud/ec2_config.py b/gillespy2/remote/cloud/ec2_config.py new file mode 100644 index 00000000..84b5329a --- /dev/null +++ b/gillespy2/remote/cloud/ec2_config.py @@ -0,0 +1,119 @@ +''' +stochss_compute.cloud.ec2_config +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +class EC2RemoteConfig: + ''' + Configure remote settings. + + :param suffix: Add a suffix to all AWS resource names. + :type suffix: str + + :param vpc_name: Identify the VPC. + :type vpc_name: str + + :param subnet_name: Identify the subnet. + :type subnet_name: str + + :param security_group_name: Identify the security group. + :type security_group_name: str + + :param server_name: Identify the server. + :type server_name: str + + :param key_name: Identify the AWS EC2 KeyPair. + :type key_name: str + + :param api_port: Port to serve from. + :type api_port: int + + :param region: Region to point to, like 'us-east-1' See `here `_. + :type region: str + + :param ami: Custom AMI to use, like 'ami-09d3b3274b6c5d4aa'. See `here `_. + :type ami: str + ''' + _AMIS = { + 'us-east-1': 'ami-09d3b3274b6c5d4aa', + 'us-east-2': 'ami-089a545a9ed9893b6', + 'us-west-1': 'ami-017c001a88dd93847', + 'us-west-2': 'ami-0d593311db5abb72b', + } + + def __init__(self, + suffix=None, + vpc_name='sssc-vpc', + subnet_name='sssc-subnet', + security_group_name='sssc-sg', + server_name='sssc-server', + key_name='sssc-server-ssh-key', + api_port=29681, + region=None, + ami=None, + ): + if suffix is not None: + suffix = f'-{suffix}' + else: + suffix = '' + + self.vpc_name = vpc_name + suffix + self.subnet_name = subnet_name + suffix + self.security_group_name = security_group_name + suffix + self.server_name = server_name + suffix + self.key_name = key_name + suffix + self.api_port = api_port + self.region = region + self.ami = ami + + +class EC2LocalConfig: + ''' + Configure local settings. + + :param key_dir: Path to a directory to store SSH key. + :type key_dir: str + + :param key_name: Name for the file. + :type key_name: str + + :param status_file: Path to a file to write instance status. Writes status to top line of that file. + :type status_file: str + + :param key_type: ed25519 or rsa + :type key_type: str + + :param key_format: pem or ppk + :type key_format: str + ''' + + def __init__(self, + key_dir='./.sssc', + key_name='sssc-server-ssh-key', + status_file=None, + key_type='ed25519', + key_format='pem', + ): + self.key_dir = key_dir + self._key_filename = f'{key_name}.{key_format}' + self.key_type = key_type + self.key_format = key_format + self.key_path = os.path.abspath( + os.path.join(self.key_dir, self._key_filename)) + self.status_file = status_file diff --git a/gillespy2/remote/cloud/exceptions.py b/gillespy2/remote/cloud/exceptions.py new file mode 100644 index 00000000..d2252207 --- /dev/null +++ b/gillespy2/remote/cloud/exceptions.py @@ -0,0 +1,41 @@ +''' +stochss_compute.cloud.exceptions +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +class ResourceException(Exception): + ''' + Misconfigured or out-of-date resources detected in the cluster setup. + ''' + def __init__(self, *args: object) -> None: + super().__init__(*args) + print('Missing or misconfigured resources.') + +class EC2ImportException(Exception): + ''' + Some extra dependencies are required for EC2. + ''' + def __init__(self, *args: object) -> None: + super().__init__(*args) + print('StochSS-Compute on EC2 requires boto3 and paramiko to be installed.\nTry: pip install boto3 paramiko') + +class EC2Exception(Exception): + ''' + General exception class. + ''' + def __init__(self, *args: object) -> None: + super().__init__(*args) diff --git a/gillespy2/remote/core/__init__.py b/gillespy2/remote/core/__init__.py new file mode 100644 index 00000000..6ae6b450 --- /dev/null +++ b/gillespy2/remote/core/__init__.py @@ -0,0 +1,22 @@ +''' +stochss_compute.core +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .errors import RemoteSimulationError +from .remote_results import RemoteResults +from .remote_simulation import RemoteSimulation diff --git a/gillespy2/remote/core/errors.py b/gillespy2/remote/core/errors.py new file mode 100644 index 00000000..92ea9e0c --- /dev/null +++ b/gillespy2/remote/core/errors.py @@ -0,0 +1,29 @@ +''' +stochss_compute.core.errors +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +class RemoteSimulationError(Exception): + ''' + General simulation error. + ''' + +class CacheError(Exception): + ''' + General cache error. + ''' + \ No newline at end of file diff --git a/gillespy2/remote/core/exceptions.py b/gillespy2/remote/core/exceptions.py new file mode 100644 index 00000000..40a88219 --- /dev/null +++ b/gillespy2/remote/core/exceptions.py @@ -0,0 +1,24 @@ +''' +stochss_compute.core.exceptions +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +class PRNGCollision(Exception): + ''' + ...Lucky??? + ''' + \ No newline at end of file diff --git a/gillespy2/remote/core/log_config.py b/gillespy2/remote/core/log_config.py new file mode 100644 index 00000000..3aeb7afe --- /dev/null +++ b/gillespy2/remote/core/log_config.py @@ -0,0 +1,36 @@ +''' +stochss_compute.core.log_config + +Global Logging Configuration +''' + +from logging import getLogger + +def init_logging(name): + ''' + Call after import to initialize logs in a module file. + To follow convention, use predefined __name__. + + Like so: + + from stochss_compute.core.log_config import init_logs + log = init_logs(__name__) + + :param name: Name for the logger. Use the dot-separated module path string. + :type name: str + + :returns: A module specific logger. + :rtype: logging.Logger + ''' + logger = getLogger(name) + return logger + +def set_global_log_level(level): + ''' + Sets the root logger log level. + + :param level: NOTSET:0, DEBUG:10, INFO:20, WARNING:30, ERROR:40, CRITICAL:50, etc. + :type level: int | logging._Level + ''' + getLogger(__name__.split('.', maxsplit=1)[0]).setLevel(level) + \ No newline at end of file diff --git a/gillespy2/remote/core/messages/__init__.py b/gillespy2/remote/core/messages/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gillespy2/remote/core/messages/base.py b/gillespy2/remote/core/messages/base.py new file mode 100644 index 00000000..d7779114 --- /dev/null +++ b/gillespy2/remote/core/messages/base.py @@ -0,0 +1,52 @@ +''' +stochss_compute.core.messages +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from abc import ABC, abstractmethod + +class Request(ABC): + ''' + Base class. + ''' + @abstractmethod + def encode(self): + ''' + Encode self for http. + ''' + @staticmethod + @abstractmethod + def parse(raw_request): + ''' + Parse http for python. + ''' + +class Response(ABC): + ''' + Base class. + ''' + @abstractmethod + def encode(self): + ''' + Encode self for http. + ''' + @staticmethod + @abstractmethod + def parse(raw_response): + ''' + Parse http for python. + ''' diff --git a/gillespy2/remote/core/messages/results.py b/gillespy2/remote/core/messages/results.py new file mode 100644 index 00000000..30c6a5d5 --- /dev/null +++ b/gillespy2/remote/core/messages/results.py @@ -0,0 +1,72 @@ +''' +stochss_compute.core.messages.results +''' +from tornado.escape import json_decode +from gillespy2 import Results + +from stochss_compute.core.messages.base import Request, Response + +class ResultsRequest(Request): + ''' + Request results from the server. + + :param results_id: Hash of the SimulationRunRequest + :type results_id: str + ''' + def __init__(self, results_id): + self.results_id = results_id + def encode(self): + ''' + :returns: self.__dict__ + :rtype: dict + ''' + return self.__dict__ + @staticmethod + def parse(raw_request): + ''' + Parse HTTP request. + + :param raw_request: The request. + :type raw_request: dict[str, str] + + :returns: The decoded object. + :rtype: ResultsRequest + ''' + request_dict = json_decode(raw_request) + return ResultsRequest(request_dict['results_id']) + +class ResultsResponse(Response): + ''' + A response from the server about the Results. + + :param results: The requested Results from the cache. (JSON) + :type results: str + + ''' + def __init__(self, results = None): + self.results = results + + def encode(self): + ''' + :returns: self.__dict__ + :rtype: dict + ''' + return {'results': self.results or ''} + + @staticmethod + def parse(raw_response): + ''' + Parse HTTP response. + + :param raw_response: The response. + :type raw_response: dict[str, str] + + :returns: The decoded object. + :rtype: ResultsResponse + ''' + response_dict = json_decode(raw_response) + if response_dict['results'] != '': + results = Results.from_json(response_dict['results']) + else: + results = None + return ResultsResponse(results) \ No newline at end of file diff --git a/gillespy2/remote/core/messages/simulation_run.py b/gillespy2/remote/core/messages/simulation_run.py new file mode 100644 index 00000000..707dd226 --- /dev/null +++ b/gillespy2/remote/core/messages/simulation_run.py @@ -0,0 +1,119 @@ +''' +stochss_compute.core.messages.simulation_run +''' +from hashlib import md5 +from tornado.escape import json_decode, json_encode +from gillespy2 import Model, Results +from stochss_compute.core.messages.base import Request, Response +from stochss_compute.core.messages.status import SimStatus + +class SimulationRunRequest(Request): + ''' + A simulation request. + + :param model: A model to run. + :type model: gillespy2.Model + + :param kwargs: kwargs for the model.run() call. + :type kwargs: dict[str, Any] + ''' + def __init__(self, model,**kwargs): + self.model = model + self.kwargs = kwargs + + def encode(self): + ''' + JSON-encode model and then encode self to dict. + ''' + return {'model': self.model.to_json(), + 'kwargs': self.kwargs, + } + + @staticmethod + def parse(raw_request): + ''' + Parse HTTP request. + + :param raw_request: The request. + :type raw_request: dict[str, str] + + :returns: The decoded object. + :rtype: SimulationRunRequest + ''' + request_dict = json_decode(raw_request) + model = Model.from_json(request_dict['model']) + kwargs = request_dict['kwargs'] + return SimulationRunRequest(model, **kwargs) + + def hash(self): + ''' + Generate a unique hash of this simulation request. + Does not include number_of_trajectories in this calculation. + + :returns: md5 hex digest. + :rtype: str + ''' + anon_model_string = self.model.to_anon().to_json(encode_private=False) + popped_kwargs = {kw:self.kwargs[kw] for kw in self.kwargs if kw!='number_of_trajectories'} + # Explanation of line above: + # Take 'self.kwargs' (a dict), and add all entries to a new dictionary, + # EXCEPT the 'number_of_trajectories' key/value pair. + kwargs_string = json_encode(popped_kwargs) + request_string = f'{anon_model_string}{kwargs_string}' + _hash = md5(str.encode(request_string)).hexdigest() + return _hash + +class SimulationRunResponse(Response): + ''' + A response from the server regarding a SimulationRunRequest. + + :param status: The status of the simulation. + :type status: SimStatus + + :param error_message: Possible error message. + :type error_message: str | None + + :param results_id: Hash of the simulation request. Identifies the results. + :type results_id: str | None + + :param results: JSON-Encoded gillespy2.Results + :type results: str | None + ''' + def __init__(self, status, error_message = None, results_id = None, results = None, task_id = None): + self.status = status + self.error_message = error_message + self.results_id = results_id + self.results = results + self.task_id = task_id + + def encode(self): + ''' + Encode self to dict. + ''' + return {'status': self.status.name, + 'error_message': self.error_message or '', + 'results_id': self.results_id or '', + 'results': self.results or '', + 'task_id': self.task_id or '',} + + @staticmethod + def parse(raw_response): + ''' + Parse HTTP response. + + :param raw_response: The response. + :type raw_response: dict[str, str] + + :returns: The decoded object. + :rtype: SimulationRunResponse + ''' + response_dict = json_decode(raw_response) + status = SimStatus.from_str(response_dict['status']) + results_id = response_dict['results_id'] + error_message = response_dict['error_message'] + task_id = response_dict['task_id'] + if response_dict['results'] != '': + results = Results.from_json(response_dict['results']) + else: + results = None + return SimulationRunResponse(status, error_message, results_id, results, task_id) diff --git a/gillespy2/remote/core/messages/simulation_run_unique.py b/gillespy2/remote/core/messages/simulation_run_unique.py new file mode 100644 index 00000000..9429c6a6 --- /dev/null +++ b/gillespy2/remote/core/messages/simulation_run_unique.py @@ -0,0 +1,89 @@ +''' +stochss_compute.core.messages.simulation_run_unique +''' +from secrets import token_hex +from tornado.escape import json_decode +from gillespy2 import Model +from stochss_compute.core.messages.base import Request, Response +from stochss_compute.core.messages.status import SimStatus + +class SimulationRunUniqueRequest(Request): + ''' + A one-off simulation request identifiable by a unique key. + + :param model: A model to run. + :type model: gillespy2.Model + + :param kwargs: kwargs for the model.run() call. + :type kwargs: dict[str, Any] + ''' + def __init__(self, model, **kwargs): + self.model = model + self.kwargs = kwargs + self.unique_key = token_hex(7) + + def encode(self): + ''' + JSON-encode model and then encode self to dict. + ''' + return {'model': self.model.to_json(), + 'kwargs': self.kwargs, + 'unique_key': self.unique_key, + } + + @staticmethod + def parse(raw_request): + ''' + Parse raw HTTP request. Done server-side. + + :param raw_request: The request. + :type raw_request: dict[str, str] + + :returns: The decoded object. + :rtype: SimulationRunUniqueRequest + ''' + request_dict = json_decode(raw_request) + model = Model.from_json(request_dict['model']) + kwargs = request_dict['kwargs'] + _ = SimulationRunUniqueRequest(model, **kwargs) + _.unique_key = request_dict['unique_key'] # apply correct token (from raw request) after object construction. + return _ + +class SimulationRunUniqueResponse(Response): + ''' + A response from the server regarding a SimulationRunUniqueRequest. + + :param status: The status of the simulation. + :type status: SimStatus + + :param error_message: Possible error message. + :type error_message: str | None + + ''' + def __init__(self, status, error_message = None): + self.status = status + self.error_message = error_message + + def encode(self): + ''' + Encode self to dict. + ''' + return {'status': self.status.name, + 'error_message': self.error_message or '', + } + + @staticmethod + def parse(raw_response): + ''' + Parse HTTP response. + + :param raw_response: The response. + :type raw_response: dict[str, str] + + :returns: The decoded object. + :rtype: SimulationRunResponse + ''' + response_dict = json_decode(raw_response) + status = SimStatus.from_str(response_dict['status']) + error_message = response_dict['error_message'] + return SimulationRunUniqueResponse(status, error_message) diff --git a/gillespy2/remote/core/messages/source_ip.py b/gillespy2/remote/core/messages/source_ip.py new file mode 100644 index 00000000..71f8b789 --- /dev/null +++ b/gillespy2/remote/core/messages/source_ip.py @@ -0,0 +1,68 @@ +''' +stochss_compute.core.messages.source_ip +''' +from tornado.escape import json_decode +from stochss_compute.core.messages.base import Request, Response + +class SourceIpRequest(Request): + ''' + Restrict server access. + + :param cloud_key: Random key generated locally during launch. + :type cloud_key: str + ''' + + def __init__(self, cloud_key): + self.cloud_key = cloud_key + + def encode(self): + ''' + :returns: self.__dict__ + :rtype: dict + ''' + return self.__dict__ + + @staticmethod + def parse(raw_request): + ''' + Parse HTTP request. + + :param raw_request: The request. + :type raw_request: dict[str, str] + + :returns: The decoded object. + :rtype: SourceIpRequest + ''' + request_dict = json_decode(raw_request) + return SourceIpRequest(request_dict['cloud_key']) + +class SourceIpResponse(Response): + ''' + Response from server containing IP address of the source. + + :param source_ip: IP address of the client. + :type source_ip: str + ''' + def __init__(self, source_ip): + self.source_ip = source_ip + + def encode(self): + ''' + :returns: self.__dict__ + :rtype: dict + ''' + return self.__dict__ + + @staticmethod + def parse(raw_response): + ''' + Parses a http response and returns a python object. + + :param raw_response: A raw http SourceIpResponse from the server. + :type raw_response: str + + :returns: The decoded object. + :rtype: SourceIpResponse + ''' + response_dict = json_decode(raw_response) + return SourceIpResponse(response_dict['source_ip']) diff --git a/gillespy2/remote/core/messages/status.py b/gillespy2/remote/core/messages/status.py new file mode 100644 index 00000000..a8b5df61 --- /dev/null +++ b/gillespy2/remote/core/messages/status.py @@ -0,0 +1,107 @@ +''' +stochss_compute.core.messages.status +''' +from enum import Enum +from tornado.escape import json_decode +from stochss_compute.core.messages.base import Request, Response + +class SimStatus(Enum): + ''' + Status describing a remote simulation. + ''' + PENDING = 'The simulation is pending.' + RUNNING = 'The simulation is still running.' + READY = 'Simulation is done and results exist in the cache.' + ERROR = 'The Simulation has encountered an error.' + DOES_NOT_EXIST = 'There is no evidence of this simulation either running or on disk.' + + @staticmethod + def from_str(name): + ''' + Convert str to Enum. + ''' + if name == 'PENDING': + return SimStatus.PENDING + if name == 'RUNNING': + return SimStatus.RUNNING + if name == 'READY': + return SimStatus.READY + if name == 'ERROR': + return SimStatus.ERROR + if name == 'DOES_NOT_EXIST': + return SimStatus.DOES_NOT_EXIST + # pylint: disable=no-member + raise ValueError(f'Not a valid status.\n{SimStatus._member_names_}') + # pylint: enable=no-member + +class StatusRequest(Request): + ''' + A request for simulation status. + + :param results_id: Hash of the SimulationRunRequest + :type results_id: str + ''' + def __init__(self, results_id): + self.results_id = results_id + def encode(self): + ''' + :returns: self.__dict__ + :rtype: dict + ''' + return self.__dict__ + + @staticmethod + def parse(raw_request): + ''' + Parse HTTP request. + + :param raw_request: The request. + :type raw_request: dict[str, str] + + :returns: The decoded object. + :rtype: StatusRequest + ''' + request_dict = json_decode(raw_request) + return StatusRequest(request_dict['results_id']) + +class StatusResponse(Response): + ''' + A response from the server about simulation status. + + :param status: Status of the simulation + :type status: SimStatus + + :param message: Possible error message or otherwise + :type message: str + ''' + def __init__(self, status, message = None): + self.status = status + self.message = message + + def encode(self): + ''' + Encodes self. + :returns: self as dict + :rtype: dict[str, str] + ''' + return {'status': self.status.name, + 'message': self.message or ''} + + @staticmethod + def parse(raw_response): + ''' + Parse HTTP response. + + :param raw_response: The response. + :type raw_response: dict[str, str] + + :returns: The decoded object. + :rtype: StatusResponse + ''' + response_dict = json_decode(raw_response) + status = SimStatus.from_str(response_dict['status']) + message = response_dict['message'] + if not message: + return StatusResponse(status) + else: + return StatusResponse(status, message) \ No newline at end of file diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py new file mode 100644 index 00000000..a81a8808 --- /dev/null +++ b/gillespy2/remote/core/remote_results.py @@ -0,0 +1,138 @@ +''' +stochss_compute.core.remote_results +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from time import sleep +from gillespy2 import Results +from stochss_compute.client.endpoint import Endpoint +from stochss_compute.core.errors import RemoteSimulationError +from stochss_compute.core.messages.results import ResultsResponse +from stochss_compute.core.messages.status import StatusResponse, SimStatus + +class RemoteResults(Results): + ''' + Wrapper for a gillespy2.Results object that exists on a remote server and which is then downloaded locally. + A Results object is: A List of Trajectory objects created by a gillespy2 solver, extends the UserList object. + + These three fields must be initialized manually: id, server, n_traj, task_id. + + :param data: A list of trajectory objects. + :type data: UserList + + :param id: ID of the cached Results object. + :type id: str + + :param server: The remote instance of StochSS-Compute where the Results are cached. + :type server: stochss_compute.ComputeServer + + :param task_id: Handle for the running simulation. + :type task_id: str + ''' + # These three fields are initialized by the server + id = None + server = None + n_traj = None + task_id = None + + # pylint:disable=super-init-not-called + def __init__(self, data = None): + self._data = data + # pylint:enable=super-init-not-called + + @property + def data(self): + """ + The trajectory data. + + :returns: self._data + :rtype: UserList + """ + if None in (self.id, self.server, self.n_traj): + raise Exception('RemoteResults must have a self.id, self.server and self.n_traj.') + + if self._data is None: + self._resolve() + return self._data + + @property + def sim_status(self): + ''' + Fetch the simulation status. + + :returns: Simulation status enum as a string. + :rtype: str + ''' + return self._status().status.name + + def get_gillespy2_results(self): + """ + Get the GillesPy2 results object from the remote results. + + :returns: The generated GillesPy2 results object. + :rtype: gillespy.Results + """ + return Results(self.data) + + + @property + def is_ready(self): + """ + True if results exist in cache on the server. + + :returns: status == SimStatus.READY + :rtype: bool + """ + return self._status().status == SimStatus.READY + + def _status(self): + # Request the status of a submitted simulation. + response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, + f"/{self.id}/{self.n_traj}/{self.task_id or ''}/status") + if not response_raw.ok: + raise RemoteSimulationError(response_raw.reason) + + status_response = StatusResponse.parse(response_raw.text) + return status_response + + def _resolve(self): + status_response = self._status() + status = status_response.status + + if status == SimStatus.RUNNING: + print('Simulation is running. Downloading results when complete......') + while True: + sleep(5) + status_response = self._status() + status = status_response.status + if status != SimStatus.RUNNING: + break + + if status in (SimStatus.DOES_NOT_EXIST, SimStatus.ERROR): + raise RemoteSimulationError(status_response.message) + + if status == SimStatus.READY: + print('Results ready. Fetching.......') + if self.id == self.task_id: + response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, f"/{self.id}/results") + else: + response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, f"/{self.id}/{self.n_traj}/results") + if not response_raw.ok: + raise RemoteSimulationError(response_raw.reason) + + response = ResultsResponse.parse(response_raw.text) + self._data = response.results.data diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py new file mode 100644 index 00000000..11a4a7fd --- /dev/null +++ b/gillespy2/remote/core/remote_simulation.py @@ -0,0 +1,181 @@ +''' +stochss_compute.core.remote_simulation +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from stochss_compute.client.endpoint import Endpoint +from stochss_compute.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse +from stochss_compute.core.messages.simulation_run_unique import SimulationRunUniqueRequest, SimulationRunUniqueResponse +from stochss_compute.core.messages.status import SimStatus +from stochss_compute.core.errors import RemoteSimulationError +from stochss_compute.core.remote_results import RemoteResults + +class RemoteSimulation: + ''' + An object representing a remote gillespy2 simulation. Requires a model and a host address. + A solver type may be provided, but does not accept instantiated solvers. + + :param model: The model to simulate. + :type model: gillespy2.Model + + :param server: A server to run the simulation. Optional if host is provided. + :type server: stochss_compute.Server + + :param host: The address of a running instance of StochSS-Compute. Optional if server is provided. + :type host: str + + :param port: The port to use when connecting to the host. + Only needed if default server port is changed. Defaults to 29681. + :type port: int + + :param solver: The type of solver to use. Does not accept instantiated solvers. + :type solver: gillespy2.GillesPySolver + ''' + def __init__(self, + model, + server = None, + host: str = None, + port: int = 29681, + solver = None, + ): + + if server is not None and host is not None: + raise RemoteSimulationError('Pass a ComputeServer/Cluster object or host but not both.') + if server is None and host is None: + raise RemoteSimulationError('Pass a ComputeServer/Cluster object or host.') + if server is None and port is None: + raise RemoteSimulationError('Pass a ComputeServer/Cluster object or port.') + + if server is None: + from stochss_compute.client.compute_server import ComputeServer + self.server = ComputeServer(host, port) + else: + self.server = server + + self.model = model + + if solver is not None: + if hasattr(solver, 'is_instantiated'): + raise RemoteSimulationError( + 'RemoteSimulation does not accept an instantiated solver object. Pass a type.') + self.solver = solver + + def is_cached(self, **params): + ''' + Checks to see if a dummy simulation exists in the cache. + + :param params: Arguments for simulation. + :type params: dict[str, Any] + + :returns: If the results are cached on the server. + :rtype: bool + ''' + if "solver" in params: + if hasattr(params['solver'], 'is_instantiated'): + raise RemoteSimulationError( + 'RemoteSimulation does not accept an instantiated solver object. Pass a type.') + params["solver"] = f"{params['solver'].__module__}.{params['solver'].__qualname__}" + if self.solver is not None: + params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" + + sim_request = SimulationRunRequest(model=self.model, **params) + results_dummy = RemoteResults() + results_dummy.id = sim_request.hash() + results_dummy.server = self.server + results_dummy.n_traj = params.get('number_of_trajectories', 1) + return results_dummy.is_ready + + def run(self, ignore_cache=False, **params): + # pylint:disable=line-too-long + """ + Simulate the Model on the target ComputeServer, returning the results or a handle to a running simulation. + See `here `_. + + :param unique: When True, ignore cache completely and return always new results. + :type unique: bool + + :param params: Arguments to pass directly to the Model#run call on the server. + :type params: dict[str, Any] + + :returns: RemoteResults populated with Results if cached, otherwise and unpopulated RemoteResults + :rtype: RemoteResults + + :raises RemoteSimulationError: In the case of SimStatus.ERROR + """ + # pylint:enable=line-too-long + + if "solver" in params: + if hasattr(params['solver'], 'is_instantiated'): + raise RemoteSimulationError( + 'RemoteSimulation does not accept an instantiated solver object. Pass a type.') + params["solver"] = f"{params['solver'].__module__}.{params['solver'].__qualname__}" + if self.solver is not None: + params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" + if ignore_cache is True: + sim_request = SimulationRunUniqueRequest(self.model, **params) + return self._run_unique(sim_request) + if ignore_cache is False: + sim_request = SimulationRunRequest(self.model, **params) + return self._run(sim_request) + + def _run(self, request): + ''' + :param request: Request to send to the server. Contains Model and related arguments. + :type request: SimulationRunRequest + ''' + response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub="/run", request=request) + if not response_raw.ok: + raise Exception(response_raw.reason) + + sim_response = SimulationRunResponse.parse(response_raw.text) + + if sim_response.status == SimStatus.ERROR: + raise RemoteSimulationError(sim_response.error_message) + if sim_response.status == SimStatus.READY: + remote_results = RemoteResults(data=sim_response.results.data) + else: + remote_results = RemoteResults() + + remote_results.id = sim_response.results_id + remote_results.server = self.server + remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) + remote_results.task_id = sim_response.task_id + + return remote_results + + def _run_unique(self, request): + ''' + Ignores the cache. Gives each simulation request a unique identifier. + + :param request: Request to send to the server. Contains Model and related arguments. + :type request: SimulationRunUniqueRequest + ''' + response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub="/run/unique", request=request) + + if not response_raw.ok: + raise Exception(response_raw.reason) + sim_response = SimulationRunUniqueResponse.parse(response_raw.text) + if not sim_response.status is SimStatus.RUNNING: + raise Exception(sim_response.error_message) + # non-conforming object creation ... possible refactor needed to solve, so left in. + remote_results = RemoteResults() + remote_results.id = request.unique_key + remote_results.task_id = request.unique_key + remote_results.server = self.server + remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) + + return remote_results diff --git a/gillespy2/remote/launch.py b/gillespy2/remote/launch.py new file mode 100644 index 00000000..4e78676d --- /dev/null +++ b/gillespy2/remote/launch.py @@ -0,0 +1,152 @@ +''' +stochss_compute.launch +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +import asyncio +from argparse import ArgumentParser, Namespace +from distributed import LocalCluster +from stochss_compute.server.api import start_api + +def launch_server(): + ''' + Start the REST API. Alias to script "stochss-compute". + + `stochss-compute --help` + OR + `python -m stochss_compute.launch --help` + ''' + def _parse_args() -> Namespace: + desc = ''' + StochSS-Compute is a server and cache that anonymizes StochSS simulation data. + ''' + parser = ArgumentParser(description=desc, add_help=True, conflict_handler='resolve') + + server = parser.add_argument_group('Server') + server.add_argument("-p", "--port", default=29681, type=int, required=False, + help="The port to use for the server. Defaults to 29681.") + + cache = parser.add_argument_group('Cache') + cache.add_argument('-c', '--cache', default='cache/', required=False, + help='Path to use for the cache. Default ./cache') + cache.add_argument('--rm', '--rm-cache', default=False, required=False, + help='Whether to delete the cache upon exit. Default False.') + + dask = parser.add_argument_group('Dask') + dask.add_argument("-H", "--dask-host", default='localhost', required=False, + help="The host to use for the dask scheduler. Defaults to localhost.") + dask.add_argument("-P", "--dask-scheduler-port", default=8786, type=int, required=False, + help="The port to use for the dask scheduler. Defaults to 8786.") + return parser.parse_args() + + args = _parse_args() + asyncio.run(start_api(**args.__dict__)) + + +def launch_with_cluster(): + ''' + Start up a Dask Cluster and StochSS-Compute REST API. Alias to script "stochss-compute-cluster". + + `stochss-compute cluster --help` + OR + `python -m stochss_compute.launch cluster --help` + ''' + + def _parse_args() -> Namespace: + usage = ''' + stochss-compute-cluster -p PORT + ''' + desc = ''' + Startup script for a StochSS-Compute cluster. + StochSS-Compute is a server and cache that anonymizes StochSS simulation data. + Uses Dask, a Python parallel computing library. + ''' + parser = ArgumentParser(description=desc, add_help=True, usage=usage, + conflict_handler='resolve') + + server = parser.add_argument_group('Server') + server.add_argument("-p", "--port", default=29681, type=int, required=False, + help="The port to use for the server. Defaults to 29681.") + + cache = parser.add_argument_group('Cache') + cache.add_argument('-c', '--cache', default='cache/', required=False, + help='Path to use for the cache.') + cache.add_argument('--rm', default=False, action='store_true', required=False, + help='Whether to delete the cache upon exit. Default False.') + + dask = parser.add_argument_group('Dask') + dask.add_argument("-H", "--dask-host", default=None, required=False, + help="The host to use for the dask scheduler. Defaults to localhost.") + dask.add_argument("-P", "--dask-scheduler-port", default=0, type=int, required=False, + help="The port to use for the dask scheduler. 0 for a random port. \ + Defaults to a random port.") + dask.add_argument('-W', '--dask-n-workers', default=None, type=int, required=False, + help='Configure the number of workers. Defaults to one per core.') + dask.add_argument('-T', '--dask-threads-per-worker', default=None, required=False, type=int, + help='Configure the threads per worker. \ + Default will let Dask decide based on your CPU.') + dask.add_argument('--dask-processes', default=None, required=False, type=bool, + help='Whether to use processes (True) or threads (False). \ + Defaults to True, unless worker_class=Worker, in which case it defaults to False.') + dask.add_argument('-D', '--dask-dashboard-address', default=':8787', required=False, + help='Address on which to listen for the Bokeh diagnostics server \ + like ‘localhost:8787’ or ‘0.0.0.0:8787’. Defaults to ‘:8787’. \ + Set to None to disable the dashboard. Use ‘:0’ for a random port.') + dask.add_argument('-N', '--dask-name', default=None, required=False, + help='A name to use when printing out the cluster, defaults to type name.') + args = parser.parse_args() + return args + + + args = _parse_args() + + dask_args = {} + for (arg, value) in vars(args).items(): + if arg.startswith('dask_'): + dask_args[arg[5:]] = value + print('Launching Dask Cluster...') + cluster = LocalCluster(**dask_args) + tokens = cluster.scheduler_address.split(':') + dask_host = tokens[1][2:] + dask_port = int(tokens[2]) + print(f'Scheduler Address: <{cluster.scheduler_address}>') + for i, worker in cluster.workers.items(): + print(f'Worker {i}: {worker}') + + print(f'Dashboard Link: <{cluster.dashboard_link}>\n') + + try: + asyncio.run(start_api(port=args.port, cache=args.cache, + dask_host=dask_host, dask_scheduler_port=dask_port, rm=args.rm)) + except asyncio.exceptions.CancelledError: + pass + finally: + print('Shutting down cluster...', end='') + asyncio.run(cluster.close()) + print('OK') + +if __name__ == '__main__': + # import os + # if 'COVERAGE_PROCESS_START' in os.environ: + # import coverage + # coverage.process_startup() + if len(sys.argv) > 1: + if sys.argv[1] == 'cluster': + del sys.argv[1] + launch_with_cluster() + launch_server() diff --git a/gillespy2/remote/server/__init__.py b/gillespy2/remote/server/__init__.py new file mode 100644 index 00000000..88fd3c70 --- /dev/null +++ b/gillespy2/remote/server/__init__.py @@ -0,0 +1,20 @@ +''' +stochss_compute.server +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .api import start_api diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py new file mode 100644 index 00000000..43f5f5cb --- /dev/null +++ b/gillespy2/remote/server/api.py @@ -0,0 +1,107 @@ +''' +stochss_compute.server.api +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import asyncio +import subprocess +from logging import INFO +from tornado.web import Application +from gillespy2.remote.server.is_cached import IsCachedHandler +from gillespy2.remote.server.run import RunHandler +from gillespy2.remote.server.run_unique import SimulationRunUniqueHandler +from gillespy2.remote.server.results_unique import ResultsUniqueHandler +from gillespy2.remote.server.sourceip import SourceIpHandler +from gillespy2.remote.server.status import StatusHandler +from gillespy2.remote.server.results import ResultsHandler +from gillespy2.remote.core.log_config import init_logging, set_global_log_level +log = init_logging(__name__) + +def _make_app(dask_host, dask_scheduler_port, cache): + scheduler_address = f'{dask_host}:{dask_scheduler_port}' + return Application([ + (r"/api/v2/simulation/gillespy2/run", RunHandler, + {'scheduler_address': scheduler_address, 'cache_dir': cache}), + (r"/api/v2/simulation/gillespy2/run/unique", SimulationRunUniqueHandler, + {'scheduler_address': scheduler_address, 'cache_dir': cache}), + (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/(?P.*?)/status", + StatusHandler, {'scheduler_address': scheduler_address, 'cache_dir': cache}), + (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/results", + ResultsHandler, {'cache_dir': cache}), + (r"/api/v2/simulation/gillespy2/(?P.*?)/results", + ResultsUniqueHandler, {'cache_dir': cache}), + (r"/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached", + IsCachedHandler, {'cache_dir': cache}), + (r"/api/v2/cloud/sourceip", SourceIpHandler), + ]) + +async def start_api( + port = 29681, + cache = 'cache/', + dask_host = 'localhost', + dask_scheduler_port = 8786, + rm = False, + logging_level = INFO, + ): + """ + Start the REST API with the following arguments. + + :param port: The port to listen on. + :type port: int + + :param cache: The cache directory path. + :type cache: str + + :param dask_host: The address of the dask cluster. + :type dask_host: str + + :param dask_scheduler_port: The port of the dask cluster. + :type dask_scheduler_port: int + + :param rm: Delete the cache when exiting this program. + :type rm: bool + + :param logging_level: Set log level for stochss_compute. + :type debug: logging._Level + """ + + set_global_log_level(logging_level) + # TODO clean up lock files here + + cache_path = os.path.abspath(cache) + app = _make_app(dask_host, dask_scheduler_port, cache) + app.listen(port) + msg=''' +========================================================================= + StochSS-Compute listening on port: %(port)d + Cache directory: %(cache_path)s + Connecting to Dask scheduler at: %(dask_host)s:%(dask_scheduler_port)d +========================================================================= +''' + log.info(msg, locals()) + + try: + await asyncio.Event().wait() + except asyncio.exceptions.CancelledError as error: + log.error(error) + finally: + if rm and os.path.exists(cache_path): + log.info('Removing cache...') + subprocess.Popen(['rm', '-r', cache_path]) + log.info('Cache Removed OK') + \ No newline at end of file diff --git a/gillespy2/remote/server/cache.py b/gillespy2/remote/server/cache.py new file mode 100644 index 00000000..32c007f2 --- /dev/null +++ b/gillespy2/remote/server/cache.py @@ -0,0 +1,173 @@ +''' +Cache for StochSS-Compute +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +from json.decoder import JSONDecodeError +from datetime import datetime +from filelock import SoftFileLock +from gillespy2 import Results + +class Cache: + ''' + Cache + + :param cache_dir: The root cache directory. + :type cache_dir: str + + :param results_id: Simulation hash. + :type results_id: str + ''' + def __init__(self, cache_dir, results_id, unique=False): + if unique is True: + while cache_dir.endswith('/'): + cache_dir = cache_dir[:-1] + cache_dir = cache_dir + '/unique/' + self.results_path = os.path.join(cache_dir, f'{results_id}.results') + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + + def create(self) -> None: + ''' + Create the results file if it does not exist. + ''' + try: + with open(self.results_path, 'x', encoding='utf-8') as file: + file.close() + except FileExistsError: + pass + + def exists(self) -> bool: + ''' + Check if the results file exists. + + :returns: os.path.exists(self.results_path) + :rtype: bool + ''' + return os.path.exists(self.results_path) + + def is_empty(self) -> bool: + ''' + Check if the results are empty. + + :returns: filesize == 0 or self.exists() + :rtype: bool + ''' + lock = SoftFileLock(f'{self.results_path}.lock') + with lock: + if self.exists(): + filesize = os.path.getsize(self.results_path) + return filesize == 0 + return True + + def is_ready(self, n_traj_wanted=0) -> bool: + ''' + Check if the results are ready to be retrieved from the cache. + + :param n_traj_wanted: The number of requested trajectories. + :type: int + + :returns: n_traj_wanted <= len() + :rtype: bool + ''' + results = self.get() + if results is None or n_traj_wanted > len(results): + return False + return True + + def n_traj_needed(self, n_traj_wanted) -> int: + ''' + Calculate the difference between the number of trajectories the user has requested + and the number of trajectories currently in the cache. + + :param n_traj_wanted: The number of requested trajectories. + :type: int + + :returns: A number greater than or equal to zero. + :rtype: int + ''' + if self.is_empty(): + return n_traj_wanted + results = self.get() + if results is None: + return n_traj_wanted + diff = n_traj_wanted - len(results) + if diff > 0: + return diff + return 0 + + def n_traj_in_cache(self) -> int: + ''' + Check the number of trajectories in the cache. + + :returns: `len()` of the gillespy2.Results + :rtype: int + ''' + if self.is_empty(): + return 0 + results = self.get() + if results is not None: + return len(results) + return 0 + + def get(self) -> Results or None: + ''' + Retrieve a gillespy2.Results object from the cache or None if error. + + :returns: Results.from_json(results_json) + :rtype: gillespy2.Results or None + ''' + try: + results_json = self.read() + return Results.from_json(results_json) + except JSONDecodeError: + return None + + def read(self) -> str: + ''' + Retrieve a gillespy2.Results object as a JSON-formatted string. + + :returns: The output of reading the file. + :rtype: str + ''' + lock = SoftFileLock(f'{self.results_path}.lock') + with lock: + with open(self.results_path,'r', encoding='utf-8') as file: + return file.read() + + def save(self, results: Results) -> None: + ''' + Save a newly processed gillespy2.Results object to the cache. + + :param results: The new Results. + :type: gillespy2.Results + ''' + msg = f'{datetime.now()} | Cache | <{self.results_path}> | ' + lock = SoftFileLock(f'{self.results_path}.lock') + with lock: + with open(self.results_path, 'r+', encoding='utf-8') as file: + try: + old_results = Results.from_json(file.read()) + combined_results = results + old_results + print(msg+'Add') + file.seek(0) + file.write(combined_results.to_json()) + except JSONDecodeError: + print(msg+'New') + file.seek(0) + file.write(results.to_json()) diff --git a/gillespy2/remote/server/is_cached.py b/gillespy2/remote/server/is_cached.py new file mode 100644 index 00000000..06ad32d5 --- /dev/null +++ b/gillespy2/remote/server/is_cached.py @@ -0,0 +1,87 @@ +''' +stochss_compute.server.is_cached +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from datetime import datetime +from tornado.web import RequestHandler +from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.messages.status import SimStatus, StatusResponse +from gillespy2.remote.server.cache import Cache + +class IsCachedHandler(RequestHandler): + ''' + Endpoint that will determine if a particular simulation exists in the cache. + ''' + def initialize(self, cache_dir): + ''' + Set cache path. + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + self.cache_dir = cache_dir + + async def get(self, results_id = None, n_traj = None): + ''' + Process GET request. + + :param results_id: Hash of the simulation. + :type results_id: str + + :param n_traj: Number of trajectories to check for. + :type n_traj: str + ''' + if None in (results_id, n_traj): + raise RemoteSimulationError('Malformed request') + n_traj = int(n_traj) + cache = Cache(self.cache_dir, results_id) + print(f'\ +{datetime.now()} |\ + Source: <{self.request.remote_ip}> |\ + Cache Check |\ + <{results_id}> |\ + Trajectories: {n_traj} ') + msg = f'{datetime.now()} | <{results_id}> | Trajectories: {n_traj} | Status: ' + exists = cache.exists() + if exists: + empty = cache.is_empty() + if empty: + print(msg+SimStatus.DOES_NOT_EXIST.name) + self._respond_dne('That simulation is not currently cached.') + else: + ready = cache.is_ready(n_traj) + if ready: + print(msg+SimStatus.READY.name) + self._respond_ready() + else: + print(msg+SimStatus.DOES_NOT_EXIST.name) + self._respond_dne(f'Not enough trajectories in cache. \ + Requested: {n_traj}, Available: {cache.n_traj_in_cache()}') + else: + print(msg+SimStatus.DOES_NOT_EXIST.name) + self._respond_dne('There is no record of that simulation') + + def _respond_ready(self): + status_response = StatusResponse(SimStatus.READY) + self.write(status_response.encode()) + self.finish() + + def _respond_dne(self, msg): + status_response = StatusResponse(SimStatus.DOES_NOT_EXIST, msg) + self.write(status_response.encode()) + self.finish() diff --git a/gillespy2/remote/server/results.py b/gillespy2/remote/server/results.py new file mode 100644 index 00000000..00106548 --- /dev/null +++ b/gillespy2/remote/server/results.py @@ -0,0 +1,70 @@ +''' +stochss_compute.server.results +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from datetime import datetime +from tornado.web import RequestHandler +from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.messages.results import ResultsResponse +from gillespy2.remote.server.cache import Cache + +class ResultsHandler(RequestHandler): + ''' + Endpoint for Results objects. + ''' + def __init__(self, application, request, **kwargs): + self.cache_dir = None + super().__init__(application, request, **kwargs) + + def data_received(self, chunk: bytes): + raise NotImplementedError() + + def initialize(self, cache_dir): + ''' + Set the cache directory. + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + self.cache_dir = cache_dir + + async def get(self, results_id = None, n_traj = None): + ''' + Process GET request. + + :param results_id: Hash of the simulation. + :type results_id: str + + :param n_traj: Number of trajectories in the request. + :type n_traj: str + ''' + if '' in (results_id, n_traj) or '/' in results_id or '/' in n_traj: + self.set_status(404, reason=f'Malformed request: {self.request.uri}') + self.finish() + raise RemoteSimulationError(f'Malformed request | <{self.request.remote_ip}>') + n_traj = int(n_traj) + print(f'{datetime.now()} | <{self.request.remote_ip}> | Results Request | <{results_id}>') + cache = Cache(self.cache_dir, results_id) + if cache.is_ready(n_traj): + results = cache.read() + results_response = ResultsResponse(results) + self.write(results_response.encode()) + else: + # This should not happen! + self.set_status(404, f'Results "{results_id}" not found.') + self.finish() diff --git a/gillespy2/remote/server/results_unique.py b/gillespy2/remote/server/results_unique.py new file mode 100644 index 00000000..cf1aa173 --- /dev/null +++ b/gillespy2/remote/server/results_unique.py @@ -0,0 +1,72 @@ +''' +stochss_compute.server.results +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tornado.web import RequestHandler +from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.messages.results import ResultsResponse +from gillespy2.remote.server.cache import Cache + +from gillespy2.remote.core.log_config import init_logging +log = init_logging(__name__) + +class ResultsUniqueHandler(RequestHandler): + ''' + Endpoint for simulation-run-unique Results objects. + ''' + def __init__(self, application, request, **kwargs): + self.cache_dir = None + super().__init__(application, request, **kwargs) + + def data_received(self, chunk: bytes): + raise NotImplementedError() + + def initialize(self, cache_dir): + ''' + Set the cache directory. + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + self.cache_dir = cache_dir + + async def get(self, results_id = None): + ''' + Process GET request. + + :param results_id: Unique id + :type results_id: str + + ''' + if '' == results_id or '/' in results_id: + self.set_status(404, reason=f'Malformed request: {self.request.uri}') + self.finish() + raise RemoteSimulationError(f'Malformed request | <{self.request.remote_ip}>') + # pylint:disable=possibly-unused-variable + remote_ip = str(self.request.remote_ip) + # pylint:enable=possibly-unused-variable + log.info('<%(remote_ip)s> | Results Request | <%(results_id)s>', locals()) + cache = Cache(self.cache_dir, results_id, unique=True) + if cache.is_ready(): + results = cache.read() + results_response = ResultsResponse(results) + self.write(results_response.encode()) + else: + # This should not happen! + self.set_status(404, f'Results "{results_id}" not found.') + self.finish() diff --git a/gillespy2/remote/server/run.py b/gillespy2/remote/server/run.py new file mode 100644 index 00000000..f8389e60 --- /dev/null +++ b/gillespy2/remote/server/run.py @@ -0,0 +1,116 @@ +''' +stochss_compute.server.run +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import random +from datetime import datetime +from secrets import token_hex + +from tornado.web import RequestHandler +from tornado.ioloop import IOLoop +from distributed import Client, Future +from gillespy2.core import Results +from stochss_compute.core.messages.status import SimStatus +from stochss_compute.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse +from stochss_compute.server.cache import Cache + + +class RunHandler(RequestHandler): + ''' + Endpoint for running Gillespy2 simulations. + ''' + + scheduler_address = None + cache_dir = None + def initialize(self, scheduler_address, cache_dir): + ''' + Sets the address to the Dask scheduler and the cache directory. + + :param scheduler_address: Scheduler address. + :type scheduler_address: str + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + + self.scheduler_address = scheduler_address + self.cache_dir = cache_dir + + async def post(self): + ''' + Process simulation run request. + ''' + sim_request = SimulationRunRequest.parse(self.request.body) + sim_hash = sim_request.hash() + log_string = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Request | <{sim_hash}> | ' + cache = Cache(self.cache_dir, sim_hash) + if not cache.exists(): + cache.create() + empty = cache.is_empty() + if not empty: + # Check the number of trajectories in the request, default 1 + n_traj = sim_request.kwargs.get('number_of_trajectories', 1) + # Compare that to the number of cached trajectories + trajectories_needed = cache.n_traj_needed(n_traj) + if trajectories_needed > 0: + sim_request.kwargs['number_of_trajectories'] = trajectories_needed + print(log_string + + f'Partial cache. Running {trajectories_needed} new trajectories.') + client = Client(self.scheduler_address) + future = self._submit(sim_request, sim_hash, client) + self._return_running(sim_hash, future.key) + IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) + else: + print(log_string + 'Returning cached results.') + results = cache.get() + ret_traj = random.sample(results, n_traj) + new_results = Results(ret_traj) + new_results_json = new_results.to_json() + sim_response = SimulationRunResponse(SimStatus.READY, results_id = sim_hash, results = new_results_json) + self.write(sim_response.encode()) + self.finish() + if empty: + print(log_string + 'Results not cached. Running simulation.') + client = Client(self.scheduler_address) + future = self._submit(sim_request, sim_hash, client) + self._return_running(sim_hash, future.key) + IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) + + def _cache(self, sim_hash, future: Future, client: Client): + results = future.result() + client.close() + cache = Cache(self.cache_dir, sim_hash) + cache.save(results) + + def _submit(self, sim_request, sim_hash, client: Client): + model = sim_request.model + kwargs = sim_request.kwargs + n_traj = kwargs.get('number_of_trajectories', 1) + if "solver" in kwargs: + from pydoc import locate + kwargs["solver"] = locate(kwargs["solver"]) + + # keep client open for now! close? + key = f'{sim_hash}:{n_traj}:{token_hex(8)}' + future = client.submit(model.run, **kwargs, key=key) + return future + + def _return_running(self, results_id, task_id): + sim_response = SimulationRunResponse(SimStatus.RUNNING, results_id=results_id, task_id=task_id) + self.write(sim_response.encode()) + self.finish() diff --git a/gillespy2/remote/server/run_unique.py b/gillespy2/remote/server/run_unique.py new file mode 100644 index 00000000..7cbedc17 --- /dev/null +++ b/gillespy2/remote/server/run_unique.py @@ -0,0 +1,127 @@ +''' +stochss_compute.server.run_unique +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from datetime import datetime + +from tornado.web import RequestHandler +from tornado.ioloop import IOLoop +from distributed import Client, Future +from gillespy2.remote.core.messages.status import SimStatus +from gillespy2.remote.core.exceptions import PRNGCollision +from gillespy2.remote.core.messages.simulation_run_unique import SimulationRunUniqueRequest, SimulationRunUniqueResponse +from gillespy2.remote.server.cache import Cache + + +class SimulationRunUniqueHandler(RequestHandler): + ''' + Endpoint for running Gillespy2 simulations. + ''' + + def __init__(self, application, request, **kwargs): + self.scheduler_address = None + self.cache_dir = None + self.unique_key = None + super().__init__(application, request, **kwargs) + + def data_received(self, chunk: bytes): + raise NotImplementedError() + + def initialize(self, scheduler_address, cache_dir): + ''' + Sets the address to the Dask scheduler and the cache directory. + Creates a new directory for one-off results files identifiable by token. + + :param scheduler_address: Scheduler address. + :type scheduler_address: str + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + self.scheduler_address = scheduler_address + while cache_dir.endswith('/'): + cache_dir = cache_dir[:-1] + self.cache_dir = cache_dir + '/unique/' + + async def post(self): + ''' + Process simulation run unique POST request. + ''' + sim_request = SimulationRunUniqueRequest.parse(self.request.body) + self.unique_key = sim_request.unique_key + cache = Cache(self.cache_dir, self.unique_key) + if cache.exists(): + self.set_status(404, reason='Try again with a different key, because that one is taken.') + self.finish() + raise PRNGCollision('Try again with a different key, because that one is taken.') + cache.create() + client = Client(self.scheduler_address) + future = self._submit(sim_request, client) + log_string = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Unique Request | <{self.unique_key}> | ' + print(log_string + 'Running simulation.') + self._return_running() + IOLoop.current().run_in_executor(None, self._cache, future, client) + + def _cache(self, future, client): + ''' + Await results, close client, save to disk. + + :param future: Handle to the running simulation, to be awaited upon. + :type future: distributed.Future + + :param client: Client to the Dask scheduler. Closing here for good measure, not sure if strictly necessary. + :type client: distributed.Client + ''' + results = future.result() + client.close() + cache = Cache(self.cache_dir, self.unique_key) + cache.save(results) + + def _submit(self, sim_request, client): + ''' + Submit request to dask scheduler. + Uses pydoc.locate to convert str to solver class name object. + + :param sim_request: The user's request for a unique simulation. + :type sim_request: SimulationRunUniqueRequest + + :param client: Client to the Dask scheduler. + :type client: distributed.Client + + :returns: Handle to the running simulation and the results on the worker. + :rtype: distributed.Future + ''' + model = sim_request.model + kwargs = sim_request.kwargs + unique_key = sim_request.unique_key + if "solver" in kwargs: + # pylint:disable=import-outside-toplevel + from pydoc import locate + # pylint:enable=import-outside-toplevel + kwargs["solver"] = locate(kwargs["solver"]) + + future = client.submit(model.run, **kwargs, key=unique_key) + return future + + def _return_running(self): + ''' + Let the user know we submitted the simulation to the scheduler. + ''' + sim_response = SimulationRunUniqueResponse(SimStatus.RUNNING) + self.write(sim_response.encode()) + self.finish() diff --git a/gillespy2/remote/server/sourceip.py b/gillespy2/remote/server/sourceip.py new file mode 100644 index 00000000..a0068631 --- /dev/null +++ b/gillespy2/remote/server/sourceip.py @@ -0,0 +1,46 @@ +''' +stochss_compute.server.sourceip +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +from tornado.web import RequestHandler +from stochss_compute.core.messages.source_ip import SourceIpRequest, SourceIpResponse + +class SourceIpHandler(RequestHandler): + ''' + Responds with the IP address associated with the request. + Used only by cloud api. + ''' + + def post(self): + ''' + Process POST request. + + :returns: request.remote_ip + :rtype: str + ''' + source_ip = self.request.remote_ip + print(f'[SourceIp Request] | Source: <{source_ip}>') + source_ip_request = SourceIpRequest.parse(self.request.body) + # could possibly also check just to see if request is valid? + if source_ip_request.cloud_key == os.environ.get('CLOUD_LOCK'): + source_ip_response = SourceIpResponse(source_ip=source_ip) + self.write(source_ip_response.encode()) + else: + self.set_status(403, 'Access denied.') + self.finish() diff --git a/gillespy2/remote/server/status.py b/gillespy2/remote/server/status.py new file mode 100644 index 00000000..87c303df --- /dev/null +++ b/gillespy2/remote/server/status.py @@ -0,0 +1,157 @@ +''' +stochss_compute.server.status +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from distributed import Client +from tornado.web import RequestHandler +from stochss_compute.core.errors import RemoteSimulationError +from stochss_compute.core.messages.status import SimStatus, StatusResponse +from stochss_compute.server.cache import Cache + +from stochss_compute.core.log_config import init_logging +log = init_logging(__name__) + +class StatusHandler(RequestHandler): + ''' + Endpoint for requesting the status of a simulation. + ''' + def __init__(self, application, request, **kwargs): + self.scheduler_address = None + self.cache_dir = None + self.task_id = None + self.results_id = None + super().__init__(application, request, **kwargs) + + def data_received(self, chunk: bytes): + raise NotImplementedError() + + def initialize(self, scheduler_address, cache_dir): + ''' + Sets the address to the Dask scheduler and the cache directory. + + :param scheduler_address: Scheduler address. + :type scheduler_address: str + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + self.scheduler_address = scheduler_address + self.cache_dir = cache_dir + + async def get(self, results_id, n_traj, task_id): + ''' + Process GET request. + + :param results_id: Hash of the simulation. Required. + :type results_id: str + + :param n_traj: Number of trajectories in the request. Default 1. + :type n_traj: str + + :param task_id: ID of the running simulation. Required. + :type task_id: str + ''' + if '' in (results_id, n_traj): + self.set_status(404, reason=f'Malformed request: {self.request.uri}') + self.finish() + raise RemoteSimulationError(f'Malformed request: {self.request.uri}') + self.results_id = results_id + self.task_id = task_id + n_traj = int(n_traj) + unique = results_id == task_id + log.debug('unique: %(unique)s', locals()) + cache = Cache(self.cache_dir, results_id, unique=unique) + log_string = f'<{self.request.remote_ip}> | Results ID: <{results_id}> | Trajectories: {n_traj} | Task ID: {task_id}' + log.info(log_string) + + msg = f'<{results_id}> | <{task_id}> | Status: ' + + exists = cache.exists() + log.debug('exists: %(exists)s', locals()) + if exists: + empty = cache.is_empty() + if empty: + if self.task_id not in ('', None): + state, err = await self._check_with_scheduler() + log.info(msg + SimStatus.RUNNING.name + f' | Task: {state} | Error: {err}') + if state == 'erred': + self._respond_error(err) + else: + self._respond_running(f'Scheduler task state: {state}') + else: + log.info(msg+SimStatus.DOES_NOT_EXIST.name) + self._respond_dne() + else: + ready = cache.is_ready(n_traj) + if ready: + log.info(msg+SimStatus.READY.name) + self._respond_ready() + else: + if self.task_id not in ('', None): + state, err = await self._check_with_scheduler() + log.info(msg+SimStatus.RUNNING.name+f' | Task: {state} | error: {err}') + if state == 'erred': + self._respond_error(err) + else: + self._respond_running(f'Scheduler task state: {state}') + else: + log.info(msg+SimStatus.DOES_NOT_EXIST.name) + self._respond_dne() + else: + log.info(msg+SimStatus.DOES_NOT_EXIST.name) + self._respond_dne() + + def _respond_ready(self): + status_response = StatusResponse(SimStatus.READY) + self.write(status_response.encode()) + self.finish() + + def _respond_error(self, error_message): + status_response = StatusResponse(SimStatus.ERROR, error_message) + self.write(status_response.encode()) + self.finish() + + def _respond_dne(self): + status_response = StatusResponse(SimStatus.DOES_NOT_EXIST, 'There is no record of that simulation.') + self.write(status_response.encode()) + self.finish() + + def _respond_running(self, message): + status_response = StatusResponse(SimStatus.RUNNING, message) + self.write(status_response.encode()) + self.finish() + + async def _check_with_scheduler(self): + ''' + Ask the scheduler for information about a task. + ''' + client = Client(self.scheduler_address) + + # define function here so that it is pickle-able + def scheduler_task_state(task_id, dask_scheduler=None): + task = dask_scheduler.tasks.get(task_id) + if task is None: + return (None, None) + if task.exception_text == "": + return (task.state, None) + return (task.state, task.exception_text) + + # Do not await. Reasons. It returns sync. + ret = client.run_on_scheduler(scheduler_task_state, self.task_id) + client.close() + return ret diff --git a/setup.py b/setup.py index 37bc1291..1d44c579 100644 --- a/setup.py +++ b/setup.py @@ -104,5 +104,20 @@ 'python_libsbml', 'lxml', ], + 'remote': [ + 'distributed == 2022.12.1', + 'requests == 2.28.1', + 'filelock == 3.9.0' + ], + 'aws': [ + 'boto3 == 1.24.71', + 'paramiko == 2.11.0', + 'python-dotenv == 0.21.0' + ], + 'remote.dev': [ + 'coverage', + 'moto == 4.1.0', + ] + }, ) From 2e609b3a8416fa8d27e32a6b86cfe1f8b55a6b73 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Sun, 28 May 2023 20:00:13 -0400 Subject: [PATCH 03/54] save draft --- gillespy2/remote/__init__.py | 11 +- gillespy2/remote/__version__.py | 2 +- gillespy2/remote/client/__init__.py | 3 +- gillespy2/remote/client/compute_server.py | 2 +- gillespy2/remote/client/server.py | 4 +- gillespy2/remote/cloud/__init__.py | 2 +- gillespy2/remote/cloud/ec2.py | 17 ++- gillespy2/remote/cloud/ec2_config.py | 2 +- gillespy2/remote/cloud/exceptions.py | 2 +- gillespy2/remote/core/__init__.py | 2 +- gillespy2/remote/core/errors.py | 2 +- gillespy2/remote/core/exceptions.py | 2 +- gillespy2/remote/core/log_config.py | 4 +- gillespy2/remote/core/messages/base.py | 2 +- gillespy2/remote/core/messages/results.py | 4 +- .../remote/core/messages/simulation_run.py | 6 +- .../core/messages/simulation_run_unique.py | 6 +- gillespy2/remote/core/messages/source_ip.py | 4 +- gillespy2/remote/core/messages/status.py | 4 +- gillespy2/remote/core/remote_results.py | 12 +- gillespy2/remote/core/remote_simulation.py | 18 +-- gillespy2/remote/launch.py | 21 ++- gillespy2/remote/server/__init__.py | 2 +- gillespy2/remote/server/api.py | 23 ++-- gillespy2/remote/server/is_cached.py | 2 +- gillespy2/remote/server/results.py | 2 +- gillespy2/remote/server/results_unique.py | 2 +- gillespy2/remote/server/run.py | 127 ++++++++++-------- gillespy2/remote/server/run_cache.py | 116 ++++++++++++++++ gillespy2/remote/server/run_unique.py | 127 ------------------ gillespy2/remote/server/sourceip.py | 4 +- gillespy2/remote/server/status.py | 10 +- 32 files changed, 271 insertions(+), 276 deletions(-) create mode 100644 gillespy2/remote/server/run_cache.py delete mode 100644 gillespy2/remote/server/run_unique.py diff --git a/gillespy2/remote/__init__.py b/gillespy2/remote/__init__.py index 5317a622..db7a59bd 100644 --- a/gillespy2/remote/__init__.py +++ b/gillespy2/remote/__init__.py @@ -1,7 +1,6 @@ ''' -stochss_compute +gillespy2.remote ''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. # This program is free software: you can redistribute it and/or modify @@ -18,7 +17,7 @@ # along with this program. If not, see . -from stochss_compute.client import * -from stochss_compute.core import * -from stochss_compute.server import * -from stochss_compute.cloud import * \ No newline at end of file +from gillespy2.remote.client import * +from gillespy2.remote.core import * +from gillespy2.remote.server import * +from gillespy2.remote.cloud import * diff --git a/gillespy2/remote/__version__.py b/gillespy2/remote/__version__.py index 49f22c08..27cec9a8 100644 --- a/gillespy2/remote/__version__.py +++ b/gillespy2/remote/__version__.py @@ -1,5 +1,5 @@ ''' -stochss_compute.__version__ +gillespy2.remote.__version__ ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/client/__init__.py b/gillespy2/remote/client/__init__.py index 1c53bef9..773e0615 100644 --- a/gillespy2/remote/client/__init__.py +++ b/gillespy2/remote/client/__init__.py @@ -1,7 +1,6 @@ ''' -stochss_compute.client +gillespy2.remote.client ''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. # This program is free software: you can redistribute it and/or modify diff --git a/gillespy2/remote/client/compute_server.py b/gillespy2/remote/client/compute_server.py index 9d984f21..d3140542 100644 --- a/gillespy2/remote/client/compute_server.py +++ b/gillespy2/remote/client/compute_server.py @@ -17,7 +17,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from stochss_compute.client.server import Server +from gillespy2.remote.client.server import Server class ComputeServer(Server): ''' diff --git a/gillespy2/remote/client/server.py b/gillespy2/remote/client/server.py index ae7b6bb5..34afa519 100644 --- a/gillespy2/remote/client/server.py +++ b/gillespy2/remote/client/server.py @@ -20,8 +20,8 @@ from time import sleep from abc import ABC, abstractmethod import requests -from stochss_compute.client.endpoint import Endpoint -from stochss_compute.core.messages.base import Request +from gillespy2.remote.client.endpoint import Endpoint +from gillespy2.remote.core.messages.base import Request class Server(ABC): ''' diff --git a/gillespy2/remote/cloud/__init__.py b/gillespy2/remote/cloud/__init__.py index eb22eb6e..dfea54d0 100644 --- a/gillespy2/remote/cloud/__init__.py +++ b/gillespy2/remote/cloud/__init__.py @@ -1,5 +1,5 @@ ''' -stochss_compute.cloud +gillespy2.remote.cloud ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/cloud/ec2.py b/gillespy2/remote/cloud/ec2.py index 5d458513..8e5b2719 100644 --- a/gillespy2/remote/cloud/ec2.py +++ b/gillespy2/remote/cloud/ec2.py @@ -1,5 +1,5 @@ ''' -stochss_compute.cloud.ec2 +gillespy2.remote.cloud.ec2 ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. @@ -21,11 +21,13 @@ import logging from time import sleep from secrets import token_hex -from stochss_compute.client.server import Server -from stochss_compute.cloud.ec2_config import EC2LocalConfig, EC2RemoteConfig -from stochss_compute.core.messages.source_ip import SourceIpRequest, SourceIpResponse -from stochss_compute.cloud.exceptions import EC2ImportException, ResourceException, EC2Exception -from stochss_compute.client.endpoint import Endpoint +from gillespy2.remote.client.server import Server +from gillespy2.remote.cloud.ec2_config import EC2LocalConfig, EC2RemoteConfig +from gillespy2.remote.core.messages.source_ip import SourceIpRequest, SourceIpResponse +from gillespy2.remote.cloud.exceptions import EC2ImportException, ResourceException, EC2Exception +from gillespy2.remote.client.endpoint import Endpoint +from gillespy2.remote.core.log_config import init_logging +log = init_logging(__name__) try: import boto3 from botocore.config import Config @@ -33,7 +35,8 @@ from botocore.exceptions import ClientError from paramiko import SSHClient, AutoAddPolicy except ImportError as err: - raise EC2ImportException from err + name = __name__ + log.warn('boto3 and parkamiko are required for %(name)s') def _ec2_logger(): diff --git a/gillespy2/remote/cloud/ec2_config.py b/gillespy2/remote/cloud/ec2_config.py index 84b5329a..0fa59d26 100644 --- a/gillespy2/remote/cloud/ec2_config.py +++ b/gillespy2/remote/cloud/ec2_config.py @@ -1,5 +1,5 @@ ''' -stochss_compute.cloud.ec2_config +gillespy2.remote.cloud.ec2_config ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/cloud/exceptions.py b/gillespy2/remote/cloud/exceptions.py index d2252207..3bb21f84 100644 --- a/gillespy2/remote/cloud/exceptions.py +++ b/gillespy2/remote/cloud/exceptions.py @@ -1,5 +1,5 @@ ''' -stochss_compute.cloud.exceptions +gillespy2.remote.cloud.exceptions ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/core/__init__.py b/gillespy2/remote/core/__init__.py index 6ae6b450..22883bea 100644 --- a/gillespy2/remote/core/__init__.py +++ b/gillespy2/remote/core/__init__.py @@ -1,5 +1,5 @@ ''' -stochss_compute.core +gillespy2.remote.core ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/core/errors.py b/gillespy2/remote/core/errors.py index 92ea9e0c..97e7f645 100644 --- a/gillespy2/remote/core/errors.py +++ b/gillespy2/remote/core/errors.py @@ -1,5 +1,5 @@ ''' -stochss_compute.core.errors +gillespy2.remote.core.errors ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/core/exceptions.py b/gillespy2/remote/core/exceptions.py index 40a88219..1b4ea19a 100644 --- a/gillespy2/remote/core/exceptions.py +++ b/gillespy2/remote/core/exceptions.py @@ -1,5 +1,5 @@ ''' -stochss_compute.core.exceptions +gillespy2.remote.core.exceptions ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/core/log_config.py b/gillespy2/remote/core/log_config.py index 3aeb7afe..00b40b40 100644 --- a/gillespy2/remote/core/log_config.py +++ b/gillespy2/remote/core/log_config.py @@ -1,5 +1,5 @@ ''' -stochss_compute.core.log_config +gillespy2.remote.core.log_config Global Logging Configuration ''' @@ -13,7 +13,7 @@ def init_logging(name): Like so: - from stochss_compute.core.log_config import init_logs + from gillespy2.remote.core.log_config import init_logs log = init_logs(__name__) :param name: Name for the logger. Use the dot-separated module path string. diff --git a/gillespy2/remote/core/messages/base.py b/gillespy2/remote/core/messages/base.py index d7779114..b3d85a6f 100644 --- a/gillespy2/remote/core/messages/base.py +++ b/gillespy2/remote/core/messages/base.py @@ -1,5 +1,5 @@ ''' -stochss_compute.core.messages +gillespy2.remote.core.messages ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/core/messages/results.py b/gillespy2/remote/core/messages/results.py index 30c6a5d5..53239e19 100644 --- a/gillespy2/remote/core/messages/results.py +++ b/gillespy2/remote/core/messages/results.py @@ -1,10 +1,10 @@ ''' -stochss_compute.core.messages.results +gillespy2.remote.core.messages.results ''' from tornado.escape import json_decode from gillespy2 import Results -from stochss_compute.core.messages.base import Request, Response +from gillespy2.remote.core.messages.base import Request, Response class ResultsRequest(Request): ''' diff --git a/gillespy2/remote/core/messages/simulation_run.py b/gillespy2/remote/core/messages/simulation_run.py index 707dd226..6b25206b 100644 --- a/gillespy2/remote/core/messages/simulation_run.py +++ b/gillespy2/remote/core/messages/simulation_run.py @@ -1,11 +1,11 @@ ''' -stochss_compute.core.messages.simulation_run +gillespy2.remote.core.messages.simulation_run ''' from hashlib import md5 from tornado.escape import json_decode, json_encode from gillespy2 import Model, Results -from stochss_compute.core.messages.base import Request, Response -from stochss_compute.core.messages.status import SimStatus +from gillespy2.remote.core.messages.base import Request, Response +from gillespy2.remote.core.messages.status import SimStatus class SimulationRunRequest(Request): ''' diff --git a/gillespy2/remote/core/messages/simulation_run_unique.py b/gillespy2/remote/core/messages/simulation_run_unique.py index 9429c6a6..c825fafa 100644 --- a/gillespy2/remote/core/messages/simulation_run_unique.py +++ b/gillespy2/remote/core/messages/simulation_run_unique.py @@ -1,11 +1,11 @@ ''' -stochss_compute.core.messages.simulation_run_unique +gillespy2.remote.core.messages.simulation_run_unique ''' from secrets import token_hex from tornado.escape import json_decode from gillespy2 import Model -from stochss_compute.core.messages.base import Request, Response -from stochss_compute.core.messages.status import SimStatus +from gillespy2.remote.core.messages.base import Request, Response +from gillespy2.remote.core.messages.status import SimStatus class SimulationRunUniqueRequest(Request): ''' diff --git a/gillespy2/remote/core/messages/source_ip.py b/gillespy2/remote/core/messages/source_ip.py index 71f8b789..85819738 100644 --- a/gillespy2/remote/core/messages/source_ip.py +++ b/gillespy2/remote/core/messages/source_ip.py @@ -1,8 +1,8 @@ ''' -stochss_compute.core.messages.source_ip +gillespy2.remote.core.messages.source_ip ''' from tornado.escape import json_decode -from stochss_compute.core.messages.base import Request, Response +from gillespy2.remote.core.messages.base import Request, Response class SourceIpRequest(Request): ''' diff --git a/gillespy2/remote/core/messages/status.py b/gillespy2/remote/core/messages/status.py index a8b5df61..c74000f6 100644 --- a/gillespy2/remote/core/messages/status.py +++ b/gillespy2/remote/core/messages/status.py @@ -1,9 +1,9 @@ ''' -stochss_compute.core.messages.status +gillespy2.remote.core.messages.status ''' from enum import Enum from tornado.escape import json_decode -from stochss_compute.core.messages.base import Request, Response +from gillespy2.remote.core.messages.base import Request, Response class SimStatus(Enum): ''' diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py index a81a8808..0b4bddf2 100644 --- a/gillespy2/remote/core/remote_results.py +++ b/gillespy2/remote/core/remote_results.py @@ -1,5 +1,5 @@ ''' -stochss_compute.core.remote_results +gillespy2.remote.core.remote_results ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. @@ -19,10 +19,10 @@ from time import sleep from gillespy2 import Results -from stochss_compute.client.endpoint import Endpoint -from stochss_compute.core.errors import RemoteSimulationError -from stochss_compute.core.messages.results import ResultsResponse -from stochss_compute.core.messages.status import StatusResponse, SimStatus +from gillespy2.remote.client.endpoint import Endpoint +from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.messages.results import ResultsResponse +from gillespy2.remote.core.messages.status import StatusResponse, SimStatus class RemoteResults(Results): ''' @@ -38,7 +38,7 @@ class RemoteResults(Results): :type id: str :param server: The remote instance of StochSS-Compute where the Results are cached. - :type server: stochss_compute.ComputeServer + :type server: gillespy2.remote.ComputeServer :param task_id: Handle for the running simulation. :type task_id: str diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index 11a4a7fd..93f529e9 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -1,5 +1,5 @@ ''' -stochss_compute.core.remote_simulation +gillespy2.remote.core.remote_simulation ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. @@ -17,12 +17,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from stochss_compute.client.endpoint import Endpoint -from stochss_compute.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse -from stochss_compute.core.messages.simulation_run_unique import SimulationRunUniqueRequest, SimulationRunUniqueResponse -from stochss_compute.core.messages.status import SimStatus -from stochss_compute.core.errors import RemoteSimulationError -from stochss_compute.core.remote_results import RemoteResults +from gillespy2.remote.client.endpoint import Endpoint +from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse +from gillespy2.remote.core.messages.simulation_run_unique import SimulationRunUniqueRequest, SimulationRunUniqueResponse +from gillespy2.remote.core.messages.status import SimStatus +from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.remote_results import RemoteResults class RemoteSimulation: ''' @@ -33,7 +33,7 @@ class RemoteSimulation: :type model: gillespy2.Model :param server: A server to run the simulation. Optional if host is provided. - :type server: stochss_compute.Server + :type server: gillespy2.remote.Server :param host: The address of a running instance of StochSS-Compute. Optional if server is provided. :type host: str @@ -61,7 +61,7 @@ def __init__(self, raise RemoteSimulationError('Pass a ComputeServer/Cluster object or port.') if server is None: - from stochss_compute.client.compute_server import ComputeServer + from gillespy2.remote.client.compute_server import ComputeServer self.server = ComputeServer(host, port) else: self.server = server diff --git a/gillespy2/remote/launch.py b/gillespy2/remote/launch.py index 4e78676d..a5d105ac 100644 --- a/gillespy2/remote/launch.py +++ b/gillespy2/remote/launch.py @@ -1,7 +1,6 @@ ''' -stochss_compute.launch +gillespy2.remote.launch ''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. # This program is free software: you can redistribute it and/or modify @@ -21,15 +20,15 @@ import asyncio from argparse import ArgumentParser, Namespace from distributed import LocalCluster -from stochss_compute.server.api import start_api +from gillespy2.remote.server.api import start_api def launch_server(): ''' - Start the REST API. Alias to script "stochss-compute". + Start the REST API. Alias to script "gillespy2-remote". - `stochss-compute --help` + `gillespy2-remote --help` OR - `python -m stochss_compute.launch --help` + `python -m gillespy2.remote.launch --help` ''' def _parse_args() -> Namespace: desc = ''' @@ -62,14 +61,14 @@ def launch_with_cluster(): ''' Start up a Dask Cluster and StochSS-Compute REST API. Alias to script "stochss-compute-cluster". - `stochss-compute cluster --help` + `gillespy2-remote-cluster --help` OR - `python -m stochss_compute.launch cluster --help` + `python -m gillespy2.remote.launch cluster --help` ''' def _parse_args() -> Namespace: usage = ''' - stochss-compute-cluster -p PORT + gillespy2-remote-cluster -p PORT ''' desc = ''' Startup script for a StochSS-Compute cluster. @@ -141,10 +140,6 @@ def _parse_args() -> Namespace: print('OK') if __name__ == '__main__': - # import os - # if 'COVERAGE_PROCESS_START' in os.environ: - # import coverage - # coverage.process_startup() if len(sys.argv) > 1: if sys.argv[1] == 'cluster': del sys.argv[1] diff --git a/gillespy2/remote/server/__init__.py b/gillespy2/remote/server/__init__.py index 88fd3c70..d9f77f5d 100644 --- a/gillespy2/remote/server/__init__.py +++ b/gillespy2/remote/server/__init__.py @@ -1,5 +1,5 @@ ''' -stochss_compute.server +gillespy2.remote.server ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py index 43f5f5cb..c2b60aab 100644 --- a/gillespy2/remote/server/api.py +++ b/gillespy2/remote/server/api.py @@ -1,5 +1,5 @@ ''' -stochss_compute.server.api +gillespy2.remote.server.api ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. @@ -22,10 +22,7 @@ import subprocess from logging import INFO from tornado.web import Application -from gillespy2.remote.server.is_cached import IsCachedHandler -from gillespy2.remote.server.run import RunHandler -from gillespy2.remote.server.run_unique import SimulationRunUniqueHandler -from gillespy2.remote.server.results_unique import ResultsUniqueHandler +from gillespy2.remote.server.run import SimulationRunHandler from gillespy2.remote.server.sourceip import SourceIpHandler from gillespy2.remote.server.status import StatusHandler from gillespy2.remote.server.results import ResultsHandler @@ -35,18 +32,18 @@ def _make_app(dask_host, dask_scheduler_port, cache): scheduler_address = f'{dask_host}:{dask_scheduler_port}' return Application([ - (r"/api/v2/simulation/gillespy2/run", RunHandler, - {'scheduler_address': scheduler_address, 'cache_dir': cache}), - (r"/api/v2/simulation/gillespy2/run/unique", SimulationRunUniqueHandler, + (r"/api/v2/simulation/gillespy2/run", SimulationRunHandler, {'scheduler_address': scheduler_address, 'cache_dir': cache}), + # (r"/api/v2/simulation/gillespy2/run/unique", SimulationRunUniqueHandler, + # {'scheduler_address': scheduler_address, 'cache_dir': cache}), (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/(?P.*?)/status", StatusHandler, {'scheduler_address': scheduler_address, 'cache_dir': cache}), (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/results", ResultsHandler, {'cache_dir': cache}), - (r"/api/v2/simulation/gillespy2/(?P.*?)/results", - ResultsUniqueHandler, {'cache_dir': cache}), - (r"/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached", - IsCachedHandler, {'cache_dir': cache}), + # (r"/api/v2/simulation/gillespy2/(?P.*?)/results", + # ResultsUniqueHandler, {'cache_dir': cache}), + # (r"/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached", + # IsCachedHandler, {'cache_dir': cache}), (r"/api/v2/cloud/sourceip", SourceIpHandler), ]) @@ -76,7 +73,7 @@ async def start_api( :param rm: Delete the cache when exiting this program. :type rm: bool - :param logging_level: Set log level for stochss_compute. + :param logging_level: Set log level for gillespy2.remote. :type debug: logging._Level """ diff --git a/gillespy2/remote/server/is_cached.py b/gillespy2/remote/server/is_cached.py index 06ad32d5..b8dcb7cb 100644 --- a/gillespy2/remote/server/is_cached.py +++ b/gillespy2/remote/server/is_cached.py @@ -1,5 +1,5 @@ ''' -stochss_compute.server.is_cached +gillespy2.remote.server.is_cached ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/server/results.py b/gillespy2/remote/server/results.py index 00106548..c0ed9986 100644 --- a/gillespy2/remote/server/results.py +++ b/gillespy2/remote/server/results.py @@ -1,5 +1,5 @@ ''' -stochss_compute.server.results +gillespy2.remote.server.results ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/server/results_unique.py b/gillespy2/remote/server/results_unique.py index cf1aa173..ba5e11d0 100644 --- a/gillespy2/remote/server/results_unique.py +++ b/gillespy2/remote/server/results_unique.py @@ -1,5 +1,5 @@ ''' -stochss_compute.server.results +gillespy2.remote.server.results ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. diff --git a/gillespy2/remote/server/run.py b/gillespy2/remote/server/run.py index f8389e60..e98f9ab8 100644 --- a/gillespy2/remote/server/run.py +++ b/gillespy2/remote/server/run.py @@ -1,5 +1,5 @@ ''' -stochss_compute.server.run +gillespy2.remote.server.run ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. @@ -17,29 +17,37 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import random from datetime import datetime -from secrets import token_hex from tornado.web import RequestHandler from tornado.ioloop import IOLoop -from distributed import Client, Future -from gillespy2.core import Results -from stochss_compute.core.messages.status import SimStatus -from stochss_compute.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse -from stochss_compute.server.cache import Cache +from distributed import Client +from gillespy2.remote.core.messages.status import SimStatus +from gillespy2.remote.core.exceptions import PRNGCollision +from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse +from gillespy2.remote.server.cache import Cache +from gillespy2.remote.core.log_config import init_logging +log = init_logging(__name__) -class RunHandler(RequestHandler): +class SimulationRunHandler(RequestHandler): ''' Endpoint for running Gillespy2 simulations. ''' - scheduler_address = None - cache_dir = None + def __init__(self, application, request, **kwargs): + self.scheduler_address = None + self.cache_dir = None + self.key = None + super().__init__(application, request, **kwargs) + + def data_received(self, chunk: bytes): + raise NotImplementedError() + def initialize(self, scheduler_address, cache_dir): ''' Sets the address to the Dask scheduler and the cache directory. + Creates a new directory for one-off results files identifiable by token. :param scheduler_address: Scheduler address. :type scheduler_address: str @@ -47,70 +55,75 @@ def initialize(self, scheduler_address, cache_dir): :param cache_dir: Path to the cache. :type cache_dir: str ''' - self.scheduler_address = scheduler_address - self.cache_dir = cache_dir + while cache_dir.endswith('/'): + cache_dir = cache_dir[:-1] + self.cache_dir = cache_dir + '/run/' async def post(self): ''' - Process simulation run request. + Process simulation run POST request. ''' sim_request = SimulationRunRequest.parse(self.request.body) - sim_hash = sim_request.hash() - log_string = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Request | <{sim_hash}> | ' - cache = Cache(self.cache_dir, sim_hash) - if not cache.exists(): - cache.create() - empty = cache.is_empty() - if not empty: - # Check the number of trajectories in the request, default 1 - n_traj = sim_request.kwargs.get('number_of_trajectories', 1) - # Compare that to the number of cached trajectories - trajectories_needed = cache.n_traj_needed(n_traj) - if trajectories_needed > 0: - sim_request.kwargs['number_of_trajectories'] = trajectories_needed - print(log_string + - f'Partial cache. Running {trajectories_needed} new trajectories.') - client = Client(self.scheduler_address) - future = self._submit(sim_request, sim_hash, client) - self._return_running(sim_hash, future.key) - IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) - else: - print(log_string + 'Returning cached results.') - results = cache.get() - ret_traj = random.sample(results, n_traj) - new_results = Results(ret_traj) - new_results_json = new_results.to_json() - sim_response = SimulationRunResponse(SimStatus.READY, results_id = sim_hash, results = new_results_json) - self.write(sim_response.encode()) - self.finish() - if empty: - print(log_string + 'Results not cached. Running simulation.') - client = Client(self.scheduler_address) - future = self._submit(sim_request, sim_hash, client) - self._return_running(sim_hash, future.key) - IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) - - def _cache(self, sim_hash, future: Future, client: Client): + self.key = sim_request.key + cache = Cache(self.cache_dir, self.key) + if cache.exists(): + self.set_status(404, reason='Try again with a different key, because that one is taken.') + self.finish() + raise PRNGCollision('Try again with a different key, because that one is taken.') + cache.create() + client = Client(self.scheduler_address) + future = self._submit(sim_request, client) + log_string = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Request | <{self.key}> | ' + print(log_string + 'Running simulation.') + self._return_running() + IOLoop.current().run_in_executor(None, self._cache, future, client) + + def _cache(self, future, client): + ''' + Await results, close client, save to disk. + + :param future: Handle to the running simulation, to be awaited upon. + :type future: distributed.Future + + :param client: Client to the Dask scheduler. Closing here for good measure, not sure if strictly necessary. + :type client: distributed.Client + ''' results = future.result() client.close() - cache = Cache(self.cache_dir, sim_hash) + cache = Cache(self.cache_dir, self.key) cache.save(results) - def _submit(self, sim_request, sim_hash, client: Client): + def _submit(self, sim_request, client): + ''' + Submit request to dask scheduler. + Uses pydoc.locate to convert str to solver class name object. + + :param sim_request: The user's request for a simulation. + :type sim_request: SimulationRunRequest + + :param client: Client to the Dask scheduler. + :type client: distributed.Client + + :returns: Handle to the running simulation and the results on the worker. + :rtype: distributed.Future + ''' model = sim_request.model kwargs = sim_request.kwargs - n_traj = kwargs.get('number_of_trajectories', 1) + key = sim_request.key if "solver" in kwargs: + # pylint:disable=import-outside-toplevel from pydoc import locate + # pylint:enable=import-outside-toplevel kwargs["solver"] = locate(kwargs["solver"]) - # keep client open for now! close? - key = f'{sim_hash}:{n_traj}:{token_hex(8)}' future = client.submit(model.run, **kwargs, key=key) return future - def _return_running(self, results_id, task_id): - sim_response = SimulationRunResponse(SimStatus.RUNNING, results_id=results_id, task_id=task_id) + def _return_running(self): + ''' + Let the user know we submitted the simulation to the scheduler. + ''' + sim_response = SimulationRunResponse(SimStatus.RUNNING) self.write(sim_response.encode()) self.finish() diff --git a/gillespy2/remote/server/run_cache.py b/gillespy2/remote/server/run_cache.py new file mode 100644 index 00000000..1448547f --- /dev/null +++ b/gillespy2/remote/server/run_cache.py @@ -0,0 +1,116 @@ +''' +gillespy2.remote.server.run +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import random +from datetime import datetime +from secrets import token_hex + +from tornado.web import RequestHandler +from tornado.ioloop import IOLoop +from distributed import Client, Future +from gillespy2.core import Results +from gillespy2.remote.core.messages.status import SimStatus +from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse +from gillespy2.remote.server.cache import Cache + + +class RunHandler(RequestHandler): + ''' + Endpoint for running Gillespy2 simulations. + ''' + + scheduler_address = None + cache_dir = None + def initialize(self, scheduler_address, cache_dir): + ''' + Sets the address to the Dask scheduler and the cache directory. + + :param scheduler_address: Scheduler address. + :type scheduler_address: str + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + + self.scheduler_address = scheduler_address + self.cache_dir = cache_dir + + async def post(self): + ''' + Process simulation run request. + ''' + sim_request = SimulationRunRequest.parse(self.request.body) + sim_hash = sim_request.hash() + log_string = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Request | <{sim_hash}> | ' + cache = Cache(self.cache_dir, sim_hash) + if not cache.exists(): + cache.create() + empty = cache.is_empty() + if not empty: + # Check the number of trajectories in the request, default 1 + n_traj = sim_request.kwargs.get('number_of_trajectories', 1) + # Compare that to the number of cached trajectories + trajectories_needed = cache.n_traj_needed(n_traj) + if trajectories_needed > 0: + sim_request.kwargs['number_of_trajectories'] = trajectories_needed + print(log_string + + f'Partial cache. Running {trajectories_needed} new trajectories.') + client = Client(self.scheduler_address) + future = self._submit(sim_request, sim_hash, client) + self._return_running(sim_hash, future.key) + IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) + else: + print(log_string + 'Returning cached results.') + results = cache.get() + ret_traj = random.sample(results, n_traj) + new_results = Results(ret_traj) + new_results_json = new_results.to_json() + sim_response = SimulationRunResponse(SimStatus.READY, results_id = sim_hash, results = new_results_json) + self.write(sim_response.encode()) + self.finish() + if empty: + print(log_string + 'Results not cached. Running simulation.') + client = Client(self.scheduler_address) + future = self._submit(sim_request, sim_hash, client) + self._return_running(sim_hash, future.key) + IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) + + def _cache(self, sim_hash, future: Future, client: Client): + results = future.result() + client.close() + cache = Cache(self.cache_dir, sim_hash) + cache.save(results) + + def _submit(self, sim_request, sim_hash, client: Client): + model = sim_request.model + kwargs = sim_request.kwargs + n_traj = kwargs.get('number_of_trajectories', 1) + if "solver" in kwargs: + from pydoc import locate + kwargs["solver"] = locate(kwargs["solver"]) + + # keep client open for now! close? + key = f'{sim_hash}:{n_traj}:{token_hex(8)}' + future = client.submit(model.run, **kwargs, key=key) + return future + + def _return_running(self, results_id, task_id): + sim_response = SimulationRunResponse(SimStatus.RUNNING, results_id=results_id, task_id=task_id) + self.write(sim_response.encode()) + self.finish() diff --git a/gillespy2/remote/server/run_unique.py b/gillespy2/remote/server/run_unique.py deleted file mode 100644 index 7cbedc17..00000000 --- a/gillespy2/remote/server/run_unique.py +++ /dev/null @@ -1,127 +0,0 @@ -''' -stochss_compute.server.run_unique -''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. -# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from datetime import datetime - -from tornado.web import RequestHandler -from tornado.ioloop import IOLoop -from distributed import Client, Future -from gillespy2.remote.core.messages.status import SimStatus -from gillespy2.remote.core.exceptions import PRNGCollision -from gillespy2.remote.core.messages.simulation_run_unique import SimulationRunUniqueRequest, SimulationRunUniqueResponse -from gillespy2.remote.server.cache import Cache - - -class SimulationRunUniqueHandler(RequestHandler): - ''' - Endpoint for running Gillespy2 simulations. - ''' - - def __init__(self, application, request, **kwargs): - self.scheduler_address = None - self.cache_dir = None - self.unique_key = None - super().__init__(application, request, **kwargs) - - def data_received(self, chunk: bytes): - raise NotImplementedError() - - def initialize(self, scheduler_address, cache_dir): - ''' - Sets the address to the Dask scheduler and the cache directory. - Creates a new directory for one-off results files identifiable by token. - - :param scheduler_address: Scheduler address. - :type scheduler_address: str - - :param cache_dir: Path to the cache. - :type cache_dir: str - ''' - self.scheduler_address = scheduler_address - while cache_dir.endswith('/'): - cache_dir = cache_dir[:-1] - self.cache_dir = cache_dir + '/unique/' - - async def post(self): - ''' - Process simulation run unique POST request. - ''' - sim_request = SimulationRunUniqueRequest.parse(self.request.body) - self.unique_key = sim_request.unique_key - cache = Cache(self.cache_dir, self.unique_key) - if cache.exists(): - self.set_status(404, reason='Try again with a different key, because that one is taken.') - self.finish() - raise PRNGCollision('Try again with a different key, because that one is taken.') - cache.create() - client = Client(self.scheduler_address) - future = self._submit(sim_request, client) - log_string = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Unique Request | <{self.unique_key}> | ' - print(log_string + 'Running simulation.') - self._return_running() - IOLoop.current().run_in_executor(None, self._cache, future, client) - - def _cache(self, future, client): - ''' - Await results, close client, save to disk. - - :param future: Handle to the running simulation, to be awaited upon. - :type future: distributed.Future - - :param client: Client to the Dask scheduler. Closing here for good measure, not sure if strictly necessary. - :type client: distributed.Client - ''' - results = future.result() - client.close() - cache = Cache(self.cache_dir, self.unique_key) - cache.save(results) - - def _submit(self, sim_request, client): - ''' - Submit request to dask scheduler. - Uses pydoc.locate to convert str to solver class name object. - - :param sim_request: The user's request for a unique simulation. - :type sim_request: SimulationRunUniqueRequest - - :param client: Client to the Dask scheduler. - :type client: distributed.Client - - :returns: Handle to the running simulation and the results on the worker. - :rtype: distributed.Future - ''' - model = sim_request.model - kwargs = sim_request.kwargs - unique_key = sim_request.unique_key - if "solver" in kwargs: - # pylint:disable=import-outside-toplevel - from pydoc import locate - # pylint:enable=import-outside-toplevel - kwargs["solver"] = locate(kwargs["solver"]) - - future = client.submit(model.run, **kwargs, key=unique_key) - return future - - def _return_running(self): - ''' - Let the user know we submitted the simulation to the scheduler. - ''' - sim_response = SimulationRunUniqueResponse(SimStatus.RUNNING) - self.write(sim_response.encode()) - self.finish() diff --git a/gillespy2/remote/server/sourceip.py b/gillespy2/remote/server/sourceip.py index a0068631..d968ee1e 100644 --- a/gillespy2/remote/server/sourceip.py +++ b/gillespy2/remote/server/sourceip.py @@ -1,5 +1,5 @@ ''' -stochss_compute.server.sourceip +gillespy2.remote.server.sourceip ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. @@ -19,7 +19,7 @@ import os from tornado.web import RequestHandler -from stochss_compute.core.messages.source_ip import SourceIpRequest, SourceIpResponse +from gillespy2.remote.core.messages.source_ip import SourceIpRequest, SourceIpResponse class SourceIpHandler(RequestHandler): ''' diff --git a/gillespy2/remote/server/status.py b/gillespy2/remote/server/status.py index 87c303df..c98c6c54 100644 --- a/gillespy2/remote/server/status.py +++ b/gillespy2/remote/server/status.py @@ -1,5 +1,5 @@ ''' -stochss_compute.server.status +gillespy2.remote.server.status ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. @@ -19,11 +19,11 @@ from distributed import Client from tornado.web import RequestHandler -from stochss_compute.core.errors import RemoteSimulationError -from stochss_compute.core.messages.status import SimStatus, StatusResponse -from stochss_compute.server.cache import Cache +from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.messages.status import SimStatus, StatusResponse +from gillespy2.remote.server.cache import Cache -from stochss_compute.core.log_config import init_logging +from gillespy2.remote.core.log_config import init_logging log = init_logging(__name__) class StatusHandler(RequestHandler): From a28eb72ba69ee440b2ec5cf7c9293bcc9a38bde5 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Sun, 28 May 2023 21:33:09 -0400 Subject: [PATCH 04/54] initial port done! --- .../remote/core/messages/simulation_run.py | 60 ++++--------- .../core/messages/simulation_run_unique.py | 89 ------------------- gillespy2/remote/core/remote_simulation.py | 55 ++++++------ gillespy2/remote/launch.py | 6 +- gillespy2/remote/server/api.py | 20 +++-- gillespy2/remote/server/cache.py | 8 +- gillespy2/remote/server/results.py | 19 ++-- .../{results_unique.py => results_cache.py} | 0 gillespy2/remote/server/run.py | 6 +- gillespy2/remote/server/status.py | 10 ++- setup.py | 7 ++ 11 files changed, 87 insertions(+), 193 deletions(-) delete mode 100644 gillespy2/remote/core/messages/simulation_run_unique.py rename gillespy2/remote/server/{results_unique.py => results_cache.py} (100%) diff --git a/gillespy2/remote/core/messages/simulation_run.py b/gillespy2/remote/core/messages/simulation_run.py index 6b25206b..8bec5727 100644 --- a/gillespy2/remote/core/messages/simulation_run.py +++ b/gillespy2/remote/core/messages/simulation_run.py @@ -1,15 +1,15 @@ ''' gillespy2.remote.core.messages.simulation_run ''' -from hashlib import md5 -from tornado.escape import json_decode, json_encode -from gillespy2 import Model, Results +from secrets import token_hex +from tornado.escape import json_decode +from gillespy2 import Model from gillespy2.remote.core.messages.base import Request, Response from gillespy2.remote.core.messages.status import SimStatus class SimulationRunRequest(Request): ''' - A simulation request. + A simulation request identifiable by a key. :param model: A model to run. :type model: gillespy2.Model @@ -17,9 +17,10 @@ class SimulationRunRequest(Request): :param kwargs: kwargs for the model.run() call. :type kwargs: dict[str, Any] ''' - def __init__(self, model,**kwargs): + def __init__(self, model, **kwargs): self.model = model self.kwargs = kwargs + self.key = token_hex(16) def encode(self): ''' @@ -27,41 +28,26 @@ def encode(self): ''' return {'model': self.model.to_json(), 'kwargs': self.kwargs, + 'key': self.key, } @staticmethod def parse(raw_request): ''' - Parse HTTP request. + Parse raw HTTP request. Done server-side. :param raw_request: The request. :type raw_request: dict[str, str] :returns: The decoded object. - :rtype: SimulationRunRequest + :rtype: SimulationRunUniqueRequest ''' request_dict = json_decode(raw_request) model = Model.from_json(request_dict['model']) kwargs = request_dict['kwargs'] - return SimulationRunRequest(model, **kwargs) - - def hash(self): - ''' - Generate a unique hash of this simulation request. - Does not include number_of_trajectories in this calculation. - - :returns: md5 hex digest. - :rtype: str - ''' - anon_model_string = self.model.to_anon().to_json(encode_private=False) - popped_kwargs = {kw:self.kwargs[kw] for kw in self.kwargs if kw!='number_of_trajectories'} - # Explanation of line above: - # Take 'self.kwargs' (a dict), and add all entries to a new dictionary, - # EXCEPT the 'number_of_trajectories' key/value pair. - kwargs_string = json_encode(popped_kwargs) - request_string = f'{anon_model_string}{kwargs_string}' - _hash = md5(str.encode(request_string)).hexdigest() - return _hash + _ = SimulationRunRequest(model, **kwargs) + _.key = request_dict['key'] # apply correct token (from raw request) after object construction. + return _ class SimulationRunResponse(Response): ''' @@ -73,18 +59,10 @@ class SimulationRunResponse(Response): :param error_message: Possible error message. :type error_message: str | None - :param results_id: Hash of the simulation request. Identifies the results. - :type results_id: str | None - - :param results: JSON-Encoded gillespy2.Results - :type results: str | None ''' - def __init__(self, status, error_message = None, results_id = None, results = None, task_id = None): + def __init__(self, status, error_message = None): self.status = status self.error_message = error_message - self.results_id = results_id - self.results = results - self.task_id = task_id def encode(self): ''' @@ -92,9 +70,7 @@ def encode(self): ''' return {'status': self.status.name, 'error_message': self.error_message or '', - 'results_id': self.results_id or '', - 'results': self.results or '', - 'task_id': self.task_id or '',} + } @staticmethod def parse(raw_response): @@ -109,11 +85,5 @@ def parse(raw_response): ''' response_dict = json_decode(raw_response) status = SimStatus.from_str(response_dict['status']) - results_id = response_dict['results_id'] error_message = response_dict['error_message'] - task_id = response_dict['task_id'] - if response_dict['results'] != '': - results = Results.from_json(response_dict['results']) - else: - results = None - return SimulationRunResponse(status, error_message, results_id, results, task_id) + return SimulationRunResponse(status, error_message) diff --git a/gillespy2/remote/core/messages/simulation_run_unique.py b/gillespy2/remote/core/messages/simulation_run_unique.py deleted file mode 100644 index c825fafa..00000000 --- a/gillespy2/remote/core/messages/simulation_run_unique.py +++ /dev/null @@ -1,89 +0,0 @@ -''' -gillespy2.remote.core.messages.simulation_run_unique -''' -from secrets import token_hex -from tornado.escape import json_decode -from gillespy2 import Model -from gillespy2.remote.core.messages.base import Request, Response -from gillespy2.remote.core.messages.status import SimStatus - -class SimulationRunUniqueRequest(Request): - ''' - A one-off simulation request identifiable by a unique key. - - :param model: A model to run. - :type model: gillespy2.Model - - :param kwargs: kwargs for the model.run() call. - :type kwargs: dict[str, Any] - ''' - def __init__(self, model, **kwargs): - self.model = model - self.kwargs = kwargs - self.unique_key = token_hex(7) - - def encode(self): - ''' - JSON-encode model and then encode self to dict. - ''' - return {'model': self.model.to_json(), - 'kwargs': self.kwargs, - 'unique_key': self.unique_key, - } - - @staticmethod - def parse(raw_request): - ''' - Parse raw HTTP request. Done server-side. - - :param raw_request: The request. - :type raw_request: dict[str, str] - - :returns: The decoded object. - :rtype: SimulationRunUniqueRequest - ''' - request_dict = json_decode(raw_request) - model = Model.from_json(request_dict['model']) - kwargs = request_dict['kwargs'] - _ = SimulationRunUniqueRequest(model, **kwargs) - _.unique_key = request_dict['unique_key'] # apply correct token (from raw request) after object construction. - return _ - -class SimulationRunUniqueResponse(Response): - ''' - A response from the server regarding a SimulationRunUniqueRequest. - - :param status: The status of the simulation. - :type status: SimStatus - - :param error_message: Possible error message. - :type error_message: str | None - - ''' - def __init__(self, status, error_message = None): - self.status = status - self.error_message = error_message - - def encode(self): - ''' - Encode self to dict. - ''' - return {'status': self.status.name, - 'error_message': self.error_message or '', - } - - @staticmethod - def parse(raw_response): - ''' - Parse HTTP response. - - :param raw_response: The response. - :type raw_response: dict[str, str] - - :returns: The decoded object. - :rtype: SimulationRunResponse - ''' - response_dict = json_decode(raw_response) - status = SimStatus.from_str(response_dict['status']) - error_message = response_dict['error_message'] - return SimulationRunUniqueResponse(status, error_message) diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index 93f529e9..5a2d54e5 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -19,7 +19,6 @@ from gillespy2.remote.client.endpoint import Endpoint from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse -from gillespy2.remote.core.messages.simulation_run_unique import SimulationRunUniqueRequest, SimulationRunUniqueResponse from gillespy2.remote.core.messages.status import SimStatus from gillespy2.remote.core.errors import RemoteSimulationError from gillespy2.remote.core.remote_results import RemoteResults @@ -126,55 +125,55 @@ def run(self, ignore_cache=False, **params): if self.solver is not None: params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" if ignore_cache is True: - sim_request = SimulationRunUniqueRequest(self.model, **params) - return self._run_unique(sim_request) + sim_request = SimulationRunRequest(self.model, **params) + return self._run(sim_request) if ignore_cache is False: sim_request = SimulationRunRequest(self.model, **params) return self._run(sim_request) - def _run(self, request): - ''' - :param request: Request to send to the server. Contains Model and related arguments. - :type request: SimulationRunRequest - ''' - response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub="/run", request=request) - if not response_raw.ok: - raise Exception(response_raw.reason) + # def _run(self, request): + # ''' + # :param request: Request to send to the server. Contains Model and related arguments. + # :type request: SimulationRunRequest + # ''' + # response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub="/run", request=request) + # if not response_raw.ok: + # raise Exception(response_raw.reason) - sim_response = SimulationRunResponse.parse(response_raw.text) + # sim_response = SimulationRunResponse.parse(response_raw.text) - if sim_response.status == SimStatus.ERROR: - raise RemoteSimulationError(sim_response.error_message) - if sim_response.status == SimStatus.READY: - remote_results = RemoteResults(data=sim_response.results.data) - else: - remote_results = RemoteResults() + # if sim_response.status == SimStatus.ERROR: + # raise RemoteSimulationError(sim_response.error_message) + # if sim_response.status == SimStatus.READY: + # remote_results = RemoteResults(data=sim_response.results.data) + # else: + # remote_results = RemoteResults() - remote_results.id = sim_response.results_id - remote_results.server = self.server - remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) - remote_results.task_id = sim_response.task_id + # remote_results.id = sim_response.results_id + # remote_results.server = self.server + # remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) + # remote_results.task_id = sim_response.task_id - return remote_results + # return remote_results - def _run_unique(self, request): + def _run(self, request): ''' Ignores the cache. Gives each simulation request a unique identifier. :param request: Request to send to the server. Contains Model and related arguments. :type request: SimulationRunUniqueRequest ''' - response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub="/run/unique", request=request) + response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub="/run", request=request) if not response_raw.ok: raise Exception(response_raw.reason) - sim_response = SimulationRunUniqueResponse.parse(response_raw.text) + sim_response = SimulationRunResponse.parse(response_raw.text) if not sim_response.status is SimStatus.RUNNING: raise Exception(sim_response.error_message) # non-conforming object creation ... possible refactor needed to solve, so left in. remote_results = RemoteResults() - remote_results.id = request.unique_key - remote_results.task_id = request.unique_key + remote_results.id = request.key + remote_results.task_id = request.key remote_results.server = self.server remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) diff --git a/gillespy2/remote/launch.py b/gillespy2/remote/launch.py index a5d105ac..3929a669 100644 --- a/gillespy2/remote/launch.py +++ b/gillespy2/remote/launch.py @@ -41,7 +41,7 @@ def _parse_args() -> Namespace: help="The port to use for the server. Defaults to 29681.") cache = parser.add_argument_group('Cache') - cache.add_argument('-c', '--cache', default='cache/', required=False, + cache.add_argument('-c', '--cache_path', default='cache/', required=False, help='Path to use for the cache. Default ./cache') cache.add_argument('--rm', '--rm-cache', default=False, required=False, help='Whether to delete the cache upon exit. Default False.') @@ -83,7 +83,7 @@ def _parse_args() -> Namespace: help="The port to use for the server. Defaults to 29681.") cache = parser.add_argument_group('Cache') - cache.add_argument('-c', '--cache', default='cache/', required=False, + cache.add_argument('-c', '--cache_path', default='cache/', required=False, help='Path to use for the cache.') cache.add_argument('--rm', default=False, action='store_true', required=False, help='Whether to delete the cache upon exit. Default False.') @@ -130,7 +130,7 @@ def _parse_args() -> Namespace: print(f'Dashboard Link: <{cluster.dashboard_link}>\n') try: - asyncio.run(start_api(port=args.port, cache=args.cache, + asyncio.run(start_api(port=args.port, cache_path=args.cache_path, dask_host=dask_host, dask_scheduler_port=dask_port, rm=args.rm)) except asyncio.exceptions.CancelledError: pass diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py index c2b60aab..108a6e72 100644 --- a/gillespy2/remote/server/api.py +++ b/gillespy2/remote/server/api.py @@ -38,10 +38,10 @@ def _make_app(dask_host, dask_scheduler_port, cache): # {'scheduler_address': scheduler_address, 'cache_dir': cache}), (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/(?P.*?)/status", StatusHandler, {'scheduler_address': scheduler_address, 'cache_dir': cache}), - (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/results", + # (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/results", + # ResultsHandler, {'cache_dir': cache}), + (r"/api/v2/simulation/gillespy2/(?P.*?)/results", ResultsHandler, {'cache_dir': cache}), - # (r"/api/v2/simulation/gillespy2/(?P.*?)/results", - # ResultsUniqueHandler, {'cache_dir': cache}), # (r"/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached", # IsCachedHandler, {'cache_dir': cache}), (r"/api/v2/cloud/sourceip", SourceIpHandler), @@ -49,7 +49,8 @@ def _make_app(dask_host, dask_scheduler_port, cache): async def start_api( port = 29681, - cache = 'cache/', + cache_trajectories = True, + cache_path = 'cache/', dask_host = 'localhost', dask_scheduler_port = 8786, rm = False, @@ -61,8 +62,11 @@ async def start_api( :param port: The port to listen on. :type port: int - :param cache: The cache directory path. - :type cache: str + :param cache_trajectories: If True, default behavior is to cache trajectories. If False, trajectory cacheing is turned off by default. Can be overridden on client side. + :type cache_trajectories: bool + + :param cache_path: The cache directory path. + :type cache_path: str :param dask_host: The address of the dask cluster. :type dask_host: str @@ -80,8 +84,8 @@ async def start_api( set_global_log_level(logging_level) # TODO clean up lock files here - cache_path = os.path.abspath(cache) - app = _make_app(dask_host, dask_scheduler_port, cache) + cache_path = os.path.abspath(cache_path) + app = _make_app(dask_host, dask_scheduler_port, cache_path) app.listen(port) msg=''' ========================================================================= diff --git a/gillespy2/remote/server/cache.py b/gillespy2/remote/server/cache.py index 32c007f2..8a9a4d0e 100644 --- a/gillespy2/remote/server/cache.py +++ b/gillespy2/remote/server/cache.py @@ -1,5 +1,5 @@ ''' -Cache for StochSS-Compute +gillespy2.remote.server.cache ''' # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. @@ -33,11 +33,7 @@ class Cache: :param results_id: Simulation hash. :type results_id: str ''' - def __init__(self, cache_dir, results_id, unique=False): - if unique is True: - while cache_dir.endswith('/'): - cache_dir = cache_dir[:-1] - cache_dir = cache_dir + '/unique/' + def __init__(self, cache_dir, results_id): self.results_path = os.path.join(cache_dir, f'{results_id}.results') if not os.path.exists(cache_dir): os.makedirs(cache_dir) diff --git a/gillespy2/remote/server/results.py b/gillespy2/remote/server/results.py index c0ed9986..d86cdcbc 100644 --- a/gillespy2/remote/server/results.py +++ b/gillespy2/remote/server/results.py @@ -1,7 +1,6 @@ ''' gillespy2.remote.server.results ''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. # This program is free software: you can redistribute it and/or modify @@ -17,12 +16,14 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from datetime import datetime from tornado.web import RequestHandler from gillespy2.remote.core.errors import RemoteSimulationError from gillespy2.remote.core.messages.results import ResultsResponse from gillespy2.remote.server.cache import Cache +from gillespy2.remote.core.log_config import init_logging +log = init_logging(__name__) + class ResultsHandler(RequestHandler): ''' Endpoint for Results objects. @@ -41,9 +42,11 @@ def initialize(self, cache_dir): :param cache_dir: Path to the cache. :type cache_dir: str ''' - self.cache_dir = cache_dir + while cache_dir.endswith('/'): + cache_dir = cache_dir[:-1] + self.cache_dir = cache_dir + '/run/' - async def get(self, results_id = None, n_traj = None): + async def get(self, results_id = None): ''' Process GET request. @@ -53,14 +56,14 @@ async def get(self, results_id = None, n_traj = None): :param n_traj: Number of trajectories in the request. :type n_traj: str ''' - if '' in (results_id, n_traj) or '/' in results_id or '/' in n_traj: + if results_id in ['', '/']: self.set_status(404, reason=f'Malformed request: {self.request.uri}') self.finish() raise RemoteSimulationError(f'Malformed request | <{self.request.remote_ip}>') - n_traj = int(n_traj) - print(f'{datetime.now()} | <{self.request.remote_ip}> | Results Request | <{results_id}>') + msg = ' <{self.request.remote_ip}> | Results Request | <{results_id}>' + log.info(msg) cache = Cache(self.cache_dir, results_id) - if cache.is_ready(n_traj): + if cache.is_ready(0): results = cache.read() results_response = ResultsResponse(results) self.write(results_response.encode()) diff --git a/gillespy2/remote/server/results_unique.py b/gillespy2/remote/server/results_cache.py similarity index 100% rename from gillespy2/remote/server/results_unique.py rename to gillespy2/remote/server/results_cache.py diff --git a/gillespy2/remote/server/run.py b/gillespy2/remote/server/run.py index e98f9ab8..9e2362d5 100644 --- a/gillespy2/remote/server/run.py +++ b/gillespy2/remote/server/run.py @@ -32,7 +32,7 @@ class SimulationRunHandler(RequestHandler): ''' - Endpoint for running Gillespy2 simulations. + Endpoint for running GillesPy2 simulations. ''' def __init__(self, application, request, **kwargs): @@ -74,8 +74,8 @@ async def post(self): cache.create() client = Client(self.scheduler_address) future = self._submit(sim_request, client) - log_string = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Request | <{self.key}> | ' - print(log_string + 'Running simulation.') + msg = '<{self.request.remote_ip}> | Simulation Run Request | <{self.key}> | Running simulation.' + log.info(msg) self._return_running() IOLoop.current().run_in_executor(None, self._cache, future, client) diff --git a/gillespy2/remote/server/status.py b/gillespy2/remote/server/status.py index c98c6c54..88d3f6e3 100644 --- a/gillespy2/remote/server/status.py +++ b/gillespy2/remote/server/status.py @@ -73,9 +73,13 @@ async def get(self, results_id, n_traj, task_id): self.results_id = results_id self.task_id = task_id n_traj = int(n_traj) - unique = results_id == task_id - log.debug('unique: %(unique)s', locals()) - cache = Cache(self.cache_dir, results_id, unique=unique) + + if results_id == task_id: + while self.cache_dir.endswith('/'): + self.cache_dir = self.cache_dir[:-1] + self.cache_dir = self.cache_dir + '/run/' + + cache = Cache(self.cache_dir, results_id) log_string = f'<{self.request.remote_ip}> | Results ID: <{results_id}> | Trajectories: {n_traj} | Task ID: {task_id}' log.info(log_string) diff --git a/setup.py b/setup.py index 1d44c579..c6ba3672 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,13 @@ "Source" : "https://github.com/StochSS/GillesPy2", }, packages = find_packages('.'), + entry_points={ + 'console_scripts': [ + 'gillespy2-remote=gillespy2.remote.launch:launch_server', + 'gillespy2-remote-cluster=gillespy2.remote.launch:launch_with_cluster', + ] + }, + include_package_data = True, install_requires = reqs, From b84ee86b6fb8e38870bb308387416be8c1616af6 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Sun, 28 May 2023 23:06:35 -0400 Subject: [PATCH 05/54] basically switch naming --- .../core/messages/simulation_run_cache.py | 117 ++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 gillespy2/remote/core/messages/simulation_run_cache.py diff --git a/gillespy2/remote/core/messages/simulation_run_cache.py b/gillespy2/remote/core/messages/simulation_run_cache.py new file mode 100644 index 00000000..ed3a6a8b --- /dev/null +++ b/gillespy2/remote/core/messages/simulation_run_cache.py @@ -0,0 +1,117 @@ +''' +gillespy2.remote.core.messages.simulation_run_cache +''' +from hashlib import md5 +from tornado.escape import json_decode, json_encode +from gillespy2 import Model, Results +from gillespy2.remote.core.messages.base import Response +from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest +from gillespy2.remote.core.messages.status import SimStatus + +class SimulationRunCacheRequest(SimulationRunRequest): + ''' + A simulation request. + + :param model: A model to run. + :type model: gillespy2.Model + + :param kwargs: kwargs for the model.run() call. + :type kwargs: dict[str, Any] + ''' + def __init__(self, model,**kwargs): + return super().__init__(model, kwargs) + + def encode(self): + ''' + JSON-encode model and then encode self to dict. + ''' + return super().encode() + + @staticmethod + def parse(raw_request): + ''' + Parse HTTP request. + + :param raw_request: The request. + :type raw_request: dict[str, str] + + :returns: The decoded object. + :rtype: SimulationRunRequest + ''' + return super().parse() + + def hash(self): + ''' + Generate a unique hash of this simulation request. + Does not include number_of_trajectories in this calculation. + + :returns: md5 hex digest. + :rtype: str + ''' + anon_model_string = self.model.to_anon().to_json(encode_private=False) + popped_kwargs = {kw:self.kwargs[kw] for kw in self.kwargs if kw!='number_of_trajectories'} + # Explanation of line above: + # Take 'self.kwargs' (a dict), and add all entries to a new dictionary, + # EXCEPT the 'number_of_trajectories' key/value pair. + kwargs_string = json_encode(popped_kwargs) + request_string = f'{anon_model_string}{kwargs_string}' + _hash = md5(str.encode(request_string)).hexdigest() + return _hash + +class SimulationRunCacheResponse(Response): + ''' + A response from the server regarding a SimulationRunCacheRequest. + + :param status: The status of the simulation. + :type status: SimStatus + + :param error_message: Possible error message. + :type error_message: str | None + + :param results_id: Hash of the simulation request. Identifies the results. + :type results_id: str | None + + :param task_id: Task ID. Handle to a running simulation. + :type task_id: str | None + + :param results: JSON-Encoded gillespy2.Results + :type results: str | None + ''' + def __init__(self, status, error_message = None, results_id = None, results = None, task_id = None): + self.status = status + self.error_message = error_message + self.results_id = results_id + self.results = results + self.task_id = task_id + + def encode(self): + ''' + Encode self to dict. + ''' + return {'status': self.status.name, + 'error_message': self.error_message or '', + 'results_id': self.results_id or '', + 'results': self.results or '', + 'task_id': self.task_id or '',} + + @staticmethod + def parse(raw_response): + ''' + Parse HTTP response. + + :param raw_response: The response. + :type raw_response: dict[str, str] + + :returns: The decoded object. + :rtype: SimulationRunResponse + ''' + response_dict = json_decode(raw_response) + status = SimStatus.from_str(response_dict['status']) + results_id = response_dict['results_id'] + error_message = response_dict['error_message'] + task_id = response_dict['task_id'] + if response_dict['results'] != '': + results = Results.from_json(response_dict['results']) + else: + results = None + return SimulationRunCacheResponse(status, error_message, results_id, results, task_id) From 4e6d9909a2e2e384ccb32328f49ed78371a57134 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Sun, 28 May 2023 23:12:26 -0400 Subject: [PATCH 06/54] decisons --- gillespy2/remote/core/remote_simulation.py | 9 ++++++--- gillespy2/remote/server/run_cache.py | 5 ++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index 5a2d54e5..1a3fdaa6 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -98,14 +98,17 @@ def is_cached(self, **params): results_dummy.n_traj = params.get('number_of_trajectories', 1) return results_dummy.is_ready - def run(self, ignore_cache=False, **params): + def run(self, namespace=None, ignore_cache=False, **params): # pylint:disable=line-too-long """ Simulate the Model on the target ComputeServer, returning the results or a handle to a running simulation. See `here `_. - :param unique: When True, ignore cache completely and return always new results. - :type unique: bool + :param namespace: TODO. + :type namespace: str + + :param ignore_cache: When True, ignore cache completely and return always new results. + :type ignore_cache: bool :param params: Arguments to pass directly to the Model#run call on the server. :type params: dict[str, Any] diff --git a/gillespy2/remote/server/run_cache.py b/gillespy2/remote/server/run_cache.py index 1448547f..21d3bd74 100644 --- a/gillespy2/remote/server/run_cache.py +++ b/gillespy2/remote/server/run_cache.py @@ -29,8 +29,7 @@ from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse from gillespy2.remote.server.cache import Cache - -class RunHandler(RequestHandler): +class RunCacheHandler(RequestHandler): ''' Endpoint for running Gillespy2 simulations. ''' @@ -57,7 +56,7 @@ async def post(self): ''' sim_request = SimulationRunRequest.parse(self.request.body) sim_hash = sim_request.hash() - log_string = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Request | <{sim_hash}> | ' + msg = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Request | <{sim_hash}> | ' cache = Cache(self.cache_dir, sim_hash) if not cache.exists(): cache.create() From 5c4bb131f90dd7130ffb26b36cd99f5cdad549e8 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 29 May 2023 15:05:57 -0400 Subject: [PATCH 07/54] typing typing typing --- .../remote/core/messages/simulation_run.py | 10 +++- .../core/messages/simulation_run_cache.py | 8 +-- gillespy2/remote/core/remote_results.py | 30 +++++++--- gillespy2/remote/core/remote_simulation.py | 56 ++++++++++--------- gillespy2/remote/server/api.py | 7 ++- .../server/{run.py => simulation_run.py} | 7 ++- .../{run_cache.py => simulation_run_cache.py} | 21 ++++--- 7 files changed, 84 insertions(+), 55 deletions(-) rename gillespy2/remote/server/{run.py => simulation_run.py} (95%) rename gillespy2/remote/server/{run_cache.py => simulation_run_cache.py} (85%) diff --git a/gillespy2/remote/core/messages/simulation_run.py b/gillespy2/remote/core/messages/simulation_run.py index 8bec5727..fe2abb5a 100644 --- a/gillespy2/remote/core/messages/simulation_run.py +++ b/gillespy2/remote/core/messages/simulation_run.py @@ -14,13 +14,17 @@ class SimulationRunRequest(Request): :param model: A model to run. :type model: gillespy2.Model + :param namespace: Optional namespace for the results. + :type namespace: str | None + :param kwargs: kwargs for the model.run() call. :type kwargs: dict[str, Any] ''' - def __init__(self, model, **kwargs): + def __init__(self, model, namespace=None, **kwargs): self.model = model self.kwargs = kwargs self.key = token_hex(16) + self.namespace = namespace def encode(self): ''' @@ -29,6 +33,7 @@ def encode(self): return {'model': self.model.to_json(), 'kwargs': self.kwargs, 'key': self.key, + 'namespace': self.namespace or '' } @staticmethod @@ -45,7 +50,8 @@ def parse(raw_request): request_dict = json_decode(raw_request) model = Model.from_json(request_dict['model']) kwargs = request_dict['kwargs'] - _ = SimulationRunRequest(model, **kwargs) + namespace = request_dict['namespace'] + _ = SimulationRunRequest(model, namespace=namespace,**kwargs) _.key = request_dict['key'] # apply correct token (from raw request) after object construction. return _ diff --git a/gillespy2/remote/core/messages/simulation_run_cache.py b/gillespy2/remote/core/messages/simulation_run_cache.py index ed3a6a8b..adf07dee 100644 --- a/gillespy2/remote/core/messages/simulation_run_cache.py +++ b/gillespy2/remote/core/messages/simulation_run_cache.py @@ -3,7 +3,7 @@ ''' from hashlib import md5 from tornado.escape import json_decode, json_encode -from gillespy2 import Model, Results +from gillespy2 import Results from gillespy2.remote.core.messages.base import Response from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest from gillespy2.remote.core.messages.status import SimStatus @@ -18,8 +18,8 @@ class SimulationRunCacheRequest(SimulationRunRequest): :param kwargs: kwargs for the model.run() call. :type kwargs: dict[str, Any] ''' - def __init__(self, model,**kwargs): - return super().__init__(model, kwargs) + def __init__(self, model, namespace=None, **kwargs): + return super().__init__(model, namespace=namespace, **kwargs) def encode(self): ''' @@ -38,7 +38,7 @@ def parse(raw_request): :returns: The decoded object. :rtype: SimulationRunRequest ''' - return super().parse() + return super().parse(raw_request) def hash(self): ''' diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py index 0b4bddf2..1f828239 100644 --- a/gillespy2/remote/core/remote_results.py +++ b/gillespy2/remote/core/remote_results.py @@ -1,7 +1,6 @@ ''' gillespy2.remote.core.remote_results ''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. # This program is free software: you can redistribute it and/or modify @@ -24,12 +23,15 @@ from gillespy2.remote.core.messages.results import ResultsResponse from gillespy2.remote.core.messages.status import StatusResponse, SimStatus +from gillespy2.remote.core.log_config import init_logging +log = init_logging(__name__) + class RemoteResults(Results): ''' Wrapper for a gillespy2.Results object that exists on a remote server and which is then downloaded locally. A Results object is: A List of Trajectory objects created by a gillespy2 solver, extends the UserList object. - These three fields must be initialized manually: id, server, n_traj, task_id. + These four fields must be initialized manually: id, server, n_traj, task_id. :param data: A list of trajectory objects. :type data: UserList @@ -43,7 +45,7 @@ class RemoteResults(Results): :param task_id: Handle for the running simulation. :type task_id: str ''' - # These three fields are initialized by the server + # These four fields are initialized after object creation. id = None server = None n_traj = None @@ -77,7 +79,8 @@ def sim_status(self): :returns: Simulation status enum as a string. :rtype: str ''' - return self._status().status.name + if self._data is not None: + return self._status().status.name def get_gillespy2_results(self): """ @@ -97,12 +100,17 @@ def is_ready(self): :returns: status == SimStatus.READY :rtype: bool """ - return self._status().status == SimStatus.READY + if self._data is not None: + return self._status().status == SimStatus.READY + return True def _status(self): + ''' + It is undefined behavior to call this function if self._data is not None. + ''' # Request the status of a submitted simulation. response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, - f"/{self.id}/{self.n_traj}/{self.task_id or ''}/status") + f"/{self.id}/{self.n_traj}/{self.task_id}/status") if not response_raw.ok: raise RemoteSimulationError(response_raw.reason) @@ -110,11 +118,14 @@ def _status(self): return status_response def _resolve(self): + ''' + It is undefined behavior to call this function if self._data is not None. + ''' status_response = self._status() status = status_response.status if status == SimStatus.RUNNING: - print('Simulation is running. Downloading results when complete......') + log.info('Simulation is running. Downloading results when complete......') while True: sleep(5) status_response = self._status() @@ -126,7 +137,7 @@ def _resolve(self): raise RemoteSimulationError(status_response.message) if status == SimStatus.READY: - print('Results ready. Fetching.......') + log.info('Results ready. Fetching.......') if self.id == self.task_id: response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, f"/{self.id}/results") else: @@ -135,4 +146,5 @@ def _resolve(self): raise RemoteSimulationError(response_raw.reason) response = ResultsResponse.parse(response_raw.text) - self._data = response.results.data + + self._data = response.results.data # Fully initialized diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index 1a3fdaa6..78ae758d 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -19,6 +19,7 @@ from gillespy2.remote.client.endpoint import Endpoint from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse +from gillespy2.remote.core.messages.simulation_run_cache import SimulationRunCacheRequest, SimulationRunCacheResponse from gillespy2.remote.core.messages.status import SimStatus from gillespy2.remote.core.errors import RemoteSimulationError from gillespy2.remote.core.remote_results import RemoteResults @@ -73,7 +74,7 @@ def __init__(self, 'RemoteSimulation does not accept an instantiated solver object. Pass a type.') self.solver = solver - def is_cached(self, **params): + def is_cached(self, namespace=None, **params): ''' Checks to see if a dummy simulation exists in the cache. @@ -91,7 +92,7 @@ def is_cached(self, **params): if self.solver is not None: params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" - sim_request = SimulationRunRequest(model=self.model, **params) + sim_request = SimulationRunCacheRequest(model=self.model, namespace=namespace, **params) results_dummy = RemoteResults() results_dummy.id = sim_request.hash() results_dummy.server = self.server @@ -104,7 +105,7 @@ def run(self, namespace=None, ignore_cache=False, **params): Simulate the Model on the target ComputeServer, returning the results or a handle to a running simulation. See `here `_. - :param namespace: TODO. + :param namespace: If provided, prepend to results path. :type namespace: str :param ignore_cache: When True, ignore cache completely and return always new results. @@ -128,38 +129,39 @@ def run(self, namespace=None, ignore_cache=False, **params): if self.solver is not None: params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" if ignore_cache is True: - sim_request = SimulationRunRequest(self.model, **params) + sim_request = SimulationRunRequest(self.model, namespace=namespace, **params) return self._run(sim_request) if ignore_cache is False: - sim_request = SimulationRunRequest(self.model, **params) - return self._run(sim_request) + sim_request = SimulationRunCacheRequest(self.model, namespace=namespace, **params) + return self._run_cache(sim_request) - # def _run(self, request): - # ''' - # :param request: Request to send to the server. Contains Model and related arguments. - # :type request: SimulationRunRequest - # ''' - # response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub="/run", request=request) - # if not response_raw.ok: - # raise Exception(response_raw.reason) + def _run_cache(self, request): + ''' + :param request: Request to send to the server. Contains Model and related arguments. + :type request: SimulationRunRequest + ''' + response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub="/run/cache", request=request) + if not response_raw.ok: + raise Exception(response_raw.reason) - # sim_response = SimulationRunResponse.parse(response_raw.text) + sim_response = SimulationRunCacheResponse.parse(response_raw.text) - # if sim_response.status == SimStatus.ERROR: - # raise RemoteSimulationError(sim_response.error_message) - # if sim_response.status == SimStatus.READY: - # remote_results = RemoteResults(data=sim_response.results.data) - # else: - # remote_results = RemoteResults() + if sim_response.status == SimStatus.ERROR: + raise RemoteSimulationError(sim_response.error_message) + # non-conforming object creation ... possible refactor needed to solve, so left in. + if sim_response.status == SimStatus.READY: + remote_results = RemoteResults(data=sim_response.results.data) + else: + remote_results = RemoteResults() - # remote_results.id = sim_response.results_id - # remote_results.server = self.server - # remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) - # remote_results.task_id = sim_response.task_id + remote_results.id = sim_response.results_id + remote_results.task_id = sim_response.task_id + remote_results.server = self.server + remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) - # return remote_results + return remote_results - def _run(self, request): + def _run(self, request, ): ''' Ignores the cache. Gives each simulation request a unique identifier. diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py index 108a6e72..99a8c656 100644 --- a/gillespy2/remote/server/api.py +++ b/gillespy2/remote/server/api.py @@ -22,7 +22,8 @@ import subprocess from logging import INFO from tornado.web import Application -from gillespy2.remote.server.run import SimulationRunHandler +from gillespy2.remote.server.simulation_run import SimulationRunHandler +from gillespy2.remote.server.simulation_run_cache import SimulationRunCacheHandler from gillespy2.remote.server.sourceip import SourceIpHandler from gillespy2.remote.server.status import StatusHandler from gillespy2.remote.server.results import ResultsHandler @@ -34,8 +35,8 @@ def _make_app(dask_host, dask_scheduler_port, cache): return Application([ (r"/api/v2/simulation/gillespy2/run", SimulationRunHandler, {'scheduler_address': scheduler_address, 'cache_dir': cache}), - # (r"/api/v2/simulation/gillespy2/run/unique", SimulationRunUniqueHandler, - # {'scheduler_address': scheduler_address, 'cache_dir': cache}), + (r"/api/v2/simulation/gillespy2/run/cache", SimulationRunCacheHandler, + {'scheduler_address': scheduler_address, 'cache_dir': cache}), (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/(?P.*?)/status", StatusHandler, {'scheduler_address': scheduler_address, 'cache_dir': cache}), # (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/results", diff --git a/gillespy2/remote/server/run.py b/gillespy2/remote/server/simulation_run.py similarity index 95% rename from gillespy2/remote/server/run.py rename to gillespy2/remote/server/simulation_run.py index 9e2362d5..c599648b 100644 --- a/gillespy2/remote/server/run.py +++ b/gillespy2/remote/server/simulation_run.py @@ -1,7 +1,6 @@ ''' gillespy2.remote.server.run ''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. # This program is free software: you can redistribute it and/or modify @@ -17,8 +16,6 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -from datetime import datetime - from tornado.web import RequestHandler from tornado.ioloop import IOLoop from distributed import Client @@ -65,6 +62,10 @@ async def post(self): Process simulation run POST request. ''' sim_request = SimulationRunRequest.parse(self.request.body) + if sim_request.namespace is not None: + while sim_request.namespace.endswith('/'): + sim_request.namespace = sim_request.namespace[:-1] + self.cache_dir = '{sim_request.namespace}/{self.cache_dir}' self.key = sim_request.key cache = Cache(self.cache_dir, self.key) if cache.exists(): diff --git a/gillespy2/remote/server/run_cache.py b/gillespy2/remote/server/simulation_run_cache.py similarity index 85% rename from gillespy2/remote/server/run_cache.py rename to gillespy2/remote/server/simulation_run_cache.py index 21d3bd74..901f297a 100644 --- a/gillespy2/remote/server/run_cache.py +++ b/gillespy2/remote/server/simulation_run_cache.py @@ -1,7 +1,6 @@ ''' gillespy2.remote.server.run ''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. # This program is free software: you can redistribute it and/or modify @@ -26,16 +25,22 @@ from distributed import Client, Future from gillespy2.core import Results from gillespy2.remote.core.messages.status import SimStatus -from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse +from gillespy2.remote.core.messages.simulation_run_cache import SimulationRunCacheRequest, SimulationRunCacheResponse from gillespy2.remote.server.cache import Cache -class RunCacheHandler(RequestHandler): +from gillespy2.remote.core.log_config import init_logging +log = init_logging(__name__) + +class SimulationRunCacheHandler(RequestHandler): ''' - Endpoint for running Gillespy2 simulations. + Endpoint for running Gillespy2 simulations as normal, + except that trajectories are cached and reused + if a particular model is run multiple times. ''' scheduler_address = None cache_dir = None + def initialize(self, scheduler_address, cache_dir): ''' Sets the address to the Dask scheduler and the cache directory. @@ -54,7 +59,9 @@ async def post(self): ''' Process simulation run request. ''' - sim_request = SimulationRunRequest.parse(self.request.body) + sim_request = SimulationRunCacheRequest.parse(self.request.body) + log.debug(sim_request.namespace) + return sim_hash = sim_request.hash() msg = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Request | <{sim_hash}> | ' cache = Cache(self.cache_dir, sim_hash) @@ -105,11 +112,11 @@ def _submit(self, sim_request, sim_hash, client: Client): kwargs["solver"] = locate(kwargs["solver"]) # keep client open for now! close? - key = f'{sim_hash}:{n_traj}:{token_hex(8)}' + key = f'{sim_hash[:-8]}:{n_traj}:{token_hex(8)}' future = client.submit(model.run, **kwargs, key=key) return future def _return_running(self, results_id, task_id): - sim_response = SimulationRunResponse(SimStatus.RUNNING, results_id=results_id, task_id=task_id) + sim_response = SimulationRunCacheResponse(SimStatus.RUNNING, results_id=results_id, task_id=task_id) self.write(sim_response.encode()) self.finish() From 9341a3ce59544cad17b8a9a6319e9769e1fb8e4f Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 29 May 2023 17:09:33 -0400 Subject: [PATCH 08/54] namespaces --- gillespy2/remote/core/messages/results.py | 13 +++- .../core/messages/simulation_run_cache.py | 4 +- gillespy2/remote/core/messages/status.py | 9 ++- gillespy2/remote/core/remote_results.py | 5 +- gillespy2/remote/server/api.py | 12 ++-- gillespy2/remote/server/cache.py | 16 ++++- gillespy2/remote/server/results.py | 4 +- gillespy2/remote/server/results_cache.py | 72 ------------------- .../remote/server/simulation_run_cache.py | 68 +++++++++++------- 9 files changed, 91 insertions(+), 112 deletions(-) delete mode 100644 gillespy2/remote/server/results_cache.py diff --git a/gillespy2/remote/core/messages/results.py b/gillespy2/remote/core/messages/results.py index 53239e19..3c75e225 100644 --- a/gillespy2/remote/core/messages/results.py +++ b/gillespy2/remote/core/messages/results.py @@ -10,17 +10,24 @@ class ResultsRequest(Request): ''' Request results from the server. - :param results_id: Hash of the SimulationRunRequest + :param results_id: Hash of the SimulationRunCacheRequest :type results_id: str + + :param namespace: Optional namespace to prepend to results directory. + :type namespace: str + ''' - def __init__(self, results_id): + def __init__(self, results_id, namespace=None): self.results_id = results_id + self.namespace = namespace + def encode(self): ''' :returns: self.__dict__ :rtype: dict ''' return self.__dict__ + @staticmethod def parse(raw_request): ''' @@ -33,7 +40,7 @@ def parse(raw_request): :rtype: ResultsRequest ''' request_dict = json_decode(raw_request) - return ResultsRequest(request_dict['results_id']) + return ResultsRequest(request_dict['results_id'], request_dict['results_id']) class ResultsResponse(Response): ''' diff --git a/gillespy2/remote/core/messages/simulation_run_cache.py b/gillespy2/remote/core/messages/simulation_run_cache.py index adf07dee..a23216c5 100644 --- a/gillespy2/remote/core/messages/simulation_run_cache.py +++ b/gillespy2/remote/core/messages/simulation_run_cache.py @@ -38,7 +38,9 @@ def parse(raw_request): :returns: The decoded object. :rtype: SimulationRunRequest ''' - return super().parse(raw_request) + # return SimulationRunCacheRequest() + _ = SimulationRunRequest.parse(raw_request) + return SimulationRunCacheRequest(_.model, namespace=_.namespace, **_.kwargs) def hash(self): ''' diff --git a/gillespy2/remote/core/messages/status.py b/gillespy2/remote/core/messages/status.py index c74000f6..7fabdbf7 100644 --- a/gillespy2/remote/core/messages/status.py +++ b/gillespy2/remote/core/messages/status.py @@ -40,9 +40,14 @@ class StatusRequest(Request): :param results_id: Hash of the SimulationRunRequest :type results_id: str + + :param namespace: Optional namespace to prepend to results directory. + :type namespace: str ''' - def __init__(self, results_id): + def __init__(self, results_id, namespace=None): self.results_id = results_id + self.namespace = namespace + def encode(self): ''' :returns: self.__dict__ @@ -62,7 +67,7 @@ def parse(raw_request): :rtype: StatusRequest ''' request_dict = json_decode(raw_request) - return StatusRequest(request_dict['results_id']) + return StatusRequest(request_dict['results_id'], request_dict['namespace']) class StatusResponse(Response): ''' diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py index 1f828239..65d701b8 100644 --- a/gillespy2/remote/core/remote_results.py +++ b/gillespy2/remote/core/remote_results.py @@ -50,6 +50,7 @@ class RemoteResults(Results): server = None n_traj = None task_id = None + namespace = None # pylint:disable=super-init-not-called def __init__(self, data = None): @@ -106,8 +107,10 @@ def is_ready(self): def _status(self): ''' - It is undefined behavior to call this function if self._data is not None. + It is undefined/illegal behavior to call this function if self._data is not None. ''' + if self._data is not None: + raise Exception('TODO Name this exception class. Cant call status on a finished simulation.') # Request the status of a submitted simulation. response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, f"/{self.id}/{self.n_traj}/{self.task_id}/status") diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py index 99a8c656..8e20cb5d 100644 --- a/gillespy2/remote/server/api.py +++ b/gillespy2/remote/server/api.py @@ -20,7 +20,7 @@ import os import asyncio import subprocess -from logging import INFO +from logging import DEBUG, INFO from tornado.web import Application from gillespy2.remote.server.simulation_run import SimulationRunHandler from gillespy2.remote.server.simulation_run_cache import SimulationRunCacheHandler @@ -35,13 +35,13 @@ def _make_app(dask_host, dask_scheduler_port, cache): return Application([ (r"/api/v2/simulation/gillespy2/run", SimulationRunHandler, {'scheduler_address': scheduler_address, 'cache_dir': cache}), + (r"/api/v2/simulation/gillespy2/(?P[0-9a-zA-Z][0-9a-zA-Z]*?)/results", + ResultsHandler, {'cache_dir': cache}), (r"/api/v2/simulation/gillespy2/run/cache", SimulationRunCacheHandler, {'scheduler_address': scheduler_address, 'cache_dir': cache}), - (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/(?P.*?)/status", + (r"/api/v2/simulation/gillespy2/(?P[0-9a-zA-Z][0-9a-zA-Z]*?)/(?P[1-9][0-9]*?)/(?P[0-9a-zA-Z][0-9a-zA-Z]*?)/status", StatusHandler, {'scheduler_address': scheduler_address, 'cache_dir': cache}), - # (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/results", - # ResultsHandler, {'cache_dir': cache}), - (r"/api/v2/simulation/gillespy2/(?P.*?)/results", + (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/results", ResultsHandler, {'cache_dir': cache}), # (r"/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached", # IsCachedHandler, {'cache_dir': cache}), @@ -55,7 +55,7 @@ async def start_api( dask_host = 'localhost', dask_scheduler_port = 8786, rm = False, - logging_level = INFO, + logging_level = DEBUG, ): """ Start the REST API with the following arguments. diff --git a/gillespy2/remote/server/cache.py b/gillespy2/remote/server/cache.py index 8a9a4d0e..6b0f8c6c 100644 --- a/gillespy2/remote/server/cache.py +++ b/gillespy2/remote/server/cache.py @@ -20,6 +20,7 @@ import os from json.decoder import JSONDecodeError from datetime import datetime +import random from filelock import SoftFileLock from gillespy2 import Results @@ -121,7 +122,7 @@ def n_traj_in_cache(self) -> int: return len(results) return 0 - def get(self) -> Results or None: + def get(self): ''' Retrieve a gillespy2.Results object from the cache or None if error. @@ -134,6 +135,19 @@ def get(self) -> Results or None: except JSONDecodeError: return None + def get_sample(self, n_traj) -> Results or None: + ''' + Retrieve a gillespy2.Results by sampling from the cache or None if error. + + :returns: Results.from_json(results_json) + :rtype: gillespy2.Results or None + ''' + results = self.get() + if results is None or len(results) <= n_traj: + return results + ret_traj = random.sample(results, n_traj) + return Results(ret_traj) + def read(self) -> str: ''' Retrieve a gillespy2.Results object as a JSON-formatted string. diff --git a/gillespy2/remote/server/results.py b/gillespy2/remote/server/results.py index d86cdcbc..b03d7a90 100644 --- a/gillespy2/remote/server/results.py +++ b/gillespy2/remote/server/results.py @@ -60,10 +60,10 @@ async def get(self, results_id = None): self.set_status(404, reason=f'Malformed request: {self.request.uri}') self.finish() raise RemoteSimulationError(f'Malformed request | <{self.request.remote_ip}>') - msg = ' <{self.request.remote_ip}> | Results Request | <{results_id}>' + msg = f' <{self.request.remote_ip}> | Results Request | <{results_id}>' log.info(msg) cache = Cache(self.cache_dir, results_id) - if cache.is_ready(0): + if cache.is_ready(): results = cache.read() results_response = ResultsResponse(results) self.write(results_response.encode()) diff --git a/gillespy2/remote/server/results_cache.py b/gillespy2/remote/server/results_cache.py deleted file mode 100644 index ba5e11d0..00000000 --- a/gillespy2/remote/server/results_cache.py +++ /dev/null @@ -1,72 +0,0 @@ -''' -gillespy2.remote.server.results -''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. -# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from tornado.web import RequestHandler -from gillespy2.remote.core.errors import RemoteSimulationError -from gillespy2.remote.core.messages.results import ResultsResponse -from gillespy2.remote.server.cache import Cache - -from gillespy2.remote.core.log_config import init_logging -log = init_logging(__name__) - -class ResultsUniqueHandler(RequestHandler): - ''' - Endpoint for simulation-run-unique Results objects. - ''' - def __init__(self, application, request, **kwargs): - self.cache_dir = None - super().__init__(application, request, **kwargs) - - def data_received(self, chunk: bytes): - raise NotImplementedError() - - def initialize(self, cache_dir): - ''' - Set the cache directory. - - :param cache_dir: Path to the cache. - :type cache_dir: str - ''' - self.cache_dir = cache_dir - - async def get(self, results_id = None): - ''' - Process GET request. - - :param results_id: Unique id - :type results_id: str - - ''' - if '' == results_id or '/' in results_id: - self.set_status(404, reason=f'Malformed request: {self.request.uri}') - self.finish() - raise RemoteSimulationError(f'Malformed request | <{self.request.remote_ip}>') - # pylint:disable=possibly-unused-variable - remote_ip = str(self.request.remote_ip) - # pylint:enable=possibly-unused-variable - log.info('<%(remote_ip)s> | Results Request | <%(results_id)s>', locals()) - cache = Cache(self.cache_dir, results_id, unique=True) - if cache.is_ready(): - results = cache.read() - results_response = ResultsResponse(results) - self.write(results_response.encode()) - else: - # This should not happen! - self.set_status(404, f'Results "{results_id}" not found.') - self.finish() diff --git a/gillespy2/remote/server/simulation_run_cache.py b/gillespy2/remote/server/simulation_run_cache.py index 901f297a..ae7d77bd 100644 --- a/gillespy2/remote/server/simulation_run_cache.py +++ b/gillespy2/remote/server/simulation_run_cache.py @@ -16,14 +16,12 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import random -from datetime import datetime +import os from secrets import token_hex from tornado.web import RequestHandler from tornado.ioloop import IOLoop -from distributed import Client, Future -from gillespy2.core import Results +from distributed import Client from gillespy2.remote.core.messages.status import SimStatus from gillespy2.remote.core.messages.simulation_run_cache import SimulationRunCacheRequest, SimulationRunCacheResponse from gillespy2.remote.server.cache import Cache @@ -60,11 +58,12 @@ async def post(self): Process simulation run request. ''' sim_request = SimulationRunCacheRequest.parse(self.request.body) - log.debug(sim_request.namespace) - return + namespace = sim_request.namespace + if namespace != '': + self.cache_dir = os.path.join(namespace, self.cache_dir) sim_hash = sim_request.hash() - msg = f'{datetime.now()} | <{self.request.remote_ip}> | Simulation Run Request | <{sim_hash}> | ' cache = Cache(self.cache_dir, sim_hash) + msg_0 = f'<{self.request.remote_ip}> | <{sim_hash}>' if not cache.exists(): cache.create() empty = cache.is_empty() @@ -75,46 +74,67 @@ async def post(self): trajectories_needed = cache.n_traj_needed(n_traj) if trajectories_needed > 0: sim_request.kwargs['number_of_trajectories'] = trajectories_needed - print(log_string + - f'Partial cache. Running {trajectories_needed} new trajectories.') + msg = f'{msg_0} | Partial cache. Running {trajectories_needed} new trajectories.' + log.info(msg) client = Client(self.scheduler_address) - future = self._submit(sim_request, sim_hash, client) + future = self._submit(sim_request, client) self._return_running(sim_hash, future.key) IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) else: - print(log_string + 'Returning cached results.') - results = cache.get() - ret_traj = random.sample(results, n_traj) - new_results = Results(ret_traj) - new_results_json = new_results.to_json() - sim_response = SimulationRunResponse(SimStatus.READY, results_id = sim_hash, results = new_results_json) + msg = f'{msg_0} | Returning cached results.' + log.info(msg) + results = cache.get_sample() + results_json = results.to_json() + sim_response = SimulationRunCacheResponse(SimStatus.READY, results_id = sim_hash, results = results_json) self.write(sim_response.encode()) self.finish() if empty: - print(log_string + 'Results not cached. Running simulation.') + msg = f'{msg_0} | Results not cached. Running simulation.' + log.info(msg) client = Client(self.scheduler_address) - future = self._submit(sim_request, sim_hash, client) + future = self._submit(sim_request, client) self._return_running(sim_hash, future.key) IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) - def _cache(self, sim_hash, future: Future, client: Client): + def _cache(self, sim_hash, future, client) -> None: + ''' + :param sim_hash: Incoming request. + :type sim_hash: SimulationRunCacheRequest + + :param future: Future that completes to gillespy2.Results. + :type future: distributed.Future + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + ''' results = future.result() client.close() cache = Cache(self.cache_dir, sim_hash) cache.save(results) - def _submit(self, sim_request, sim_hash, client: Client): + def _submit(self, sim_request, client): + ''' + :param sim_request: Incoming request. + :type sim_request: SimulationRunCacheRequest + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + :returns: Future that completes to gillespy2.Results. + :rtype: distributed.Future + ''' model = sim_request.model kwargs = sim_request.kwargs n_traj = kwargs.get('number_of_trajectories', 1) + sim_hash_tag = sim_request.hash()[:-8] + key = f'{sim_hash_tag}{n_traj}{token_hex(8)}' + if "solver" in kwargs: from pydoc import locate kwargs["solver"] = locate(kwargs["solver"]) - # keep client open for now! close? - key = f'{sim_hash[:-8]}:{n_traj}:{token_hex(8)}' - future = client.submit(model.run, **kwargs, key=key) - return future + return client.submit(model.run, **kwargs, key=key) def _return_running(self, results_id, task_id): sim_response = SimulationRunCacheResponse(SimStatus.RUNNING, results_id=results_id, task_id=task_id) From 9b7cf09df2ecaaa0de681a2cc25b338cd52d44e5 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 29 May 2023 19:11:00 -0400 Subject: [PATCH 09/54] break --- gillespy2/remote/core/messages/status.py | 10 ++++-- gillespy2/remote/core/remote_results.py | 3 +- gillespy2/remote/server/api.py | 36 +++++++++++-------- gillespy2/remote/server/simulation_run.py | 11 +++--- .../remote/server/simulation_run_cache.py | 7 ++-- gillespy2/remote/server/status.py | 10 +++--- 6 files changed, 47 insertions(+), 30 deletions(-) diff --git a/gillespy2/remote/core/messages/status.py b/gillespy2/remote/core/messages/status.py index 7fabdbf7..ed52362e 100644 --- a/gillespy2/remote/core/messages/status.py +++ b/gillespy2/remote/core/messages/status.py @@ -44,9 +44,11 @@ class StatusRequest(Request): :param namespace: Optional namespace to prepend to results directory. :type namespace: str ''' - def __init__(self, results_id, namespace=None): + def __init__(self, results_id, n_traj=None, task_id=None, namespace=None): self.results_id = results_id self.namespace = namespace + self.n_traj = n_traj + self.task_id = task_id def encode(self): ''' @@ -67,7 +69,11 @@ def parse(raw_request): :rtype: StatusRequest ''' request_dict = json_decode(raw_request) - return StatusRequest(request_dict['results_id'], request_dict['namespace']) + results_id = request_dict.get('results_id', None) + namespace = request_dict.get('namespace', None) + n_traj = request_dict.get('n_traj', None) + task_id = request_dict.get('task_id', None) + return StatusRequest(results_id, namespace, n_traj, task_id) class StatusResponse(Response): ''' diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py index 65d701b8..5e815af9 100644 --- a/gillespy2/remote/core/remote_results.py +++ b/gillespy2/remote/core/remote_results.py @@ -21,7 +21,7 @@ from gillespy2.remote.client.endpoint import Endpoint from gillespy2.remote.core.errors import RemoteSimulationError from gillespy2.remote.core.messages.results import ResultsResponse -from gillespy2.remote.core.messages.status import StatusResponse, SimStatus +from gillespy2.remote.core.messages.status import StatusRequest, StatusResponse, SimStatus from gillespy2.remote.core.log_config import init_logging log = init_logging(__name__) @@ -112,6 +112,7 @@ def _status(self): if self._data is not None: raise Exception('TODO Name this exception class. Cant call status on a finished simulation.') # Request the status of a submitted simulation. + status_request = StatusRequest(self.id, self.namespace) response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, f"/{self.id}/{self.n_traj}/{self.task_id}/status") if not response_raw.ok: diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py index 8e20cb5d..0e1a4aab 100644 --- a/gillespy2/remote/server/api.py +++ b/gillespy2/remote/server/api.py @@ -32,20 +32,28 @@ def _make_app(dask_host, dask_scheduler_port, cache): scheduler_address = f'{dask_host}:{dask_scheduler_port}' + args = {'scheduler_address': scheduler_address, + 'cache_dir': cache} + cache_arg = {'cache_dir': cache} return Application([ - (r"/api/v2/simulation/gillespy2/run", SimulationRunHandler, - {'scheduler_address': scheduler_address, 'cache_dir': cache}), - (r"/api/v2/simulation/gillespy2/(?P[0-9a-zA-Z][0-9a-zA-Z]*?)/results", - ResultsHandler, {'cache_dir': cache}), - (r"/api/v2/simulation/gillespy2/run/cache", SimulationRunCacheHandler, - {'scheduler_address': scheduler_address, 'cache_dir': cache}), - (r"/api/v2/simulation/gillespy2/(?P[0-9a-zA-Z][0-9a-zA-Z]*?)/(?P[1-9][0-9]*?)/(?P[0-9a-zA-Z][0-9a-zA-Z]*?)/status", - StatusHandler, {'scheduler_address': scheduler_address, 'cache_dir': cache}), - (r"/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/results", - ResultsHandler, {'cache_dir': cache}), - # (r"/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached", + (r'/api/v2/simulation/gillespy2/run', + SimulationRunHandler, + args), + (r'/api/v2/simulation/gillespy2/run/cache', + SimulationRunCacheHandler, + args), + (r'/api/v2/simulation/gillespy2/status', + StatusHandler, + args), + (r'/api/v2/simulation/gillespy2/(?P[0-9a-zA-Z][0-9a-zA-Z]*?)/results', + ResultsHandler, + cache_arg), + (r'/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/results', + ResultsHandler, + cache_arg), + # (r'/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached', # IsCachedHandler, {'cache_dir': cache}), - (r"/api/v2/cloud/sourceip", SourceIpHandler), + (r'/api/v2/cloud/sourceip', SourceIpHandler), ]) async def start_api( @@ -66,7 +74,7 @@ async def start_api( :param cache_trajectories: If True, default behavior is to cache trajectories. If False, trajectory cacheing is turned off by default. Can be overridden on client side. :type cache_trajectories: bool - :param cache_path: The cache directory path. + :param cache_path: The cache directory path. Do not begin with /. :type cache_path: str :param dask_host: The address of the dask cluster. @@ -85,7 +93,7 @@ async def start_api( set_global_log_level(logging_level) # TODO clean up lock files here - cache_path = os.path.abspath(cache_path) + # cache_path = os.path.abspath(cache_path) app = _make_app(dask_host, dask_scheduler_port, cache_path) app.listen(port) msg=''' diff --git a/gillespy2/remote/server/simulation_run.py b/gillespy2/remote/server/simulation_run.py index c599648b..c618fe2e 100644 --- a/gillespy2/remote/server/simulation_run.py +++ b/gillespy2/remote/server/simulation_run.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os from tornado.web import RequestHandler from tornado.ioloop import IOLoop from distributed import Client @@ -53,9 +54,7 @@ def initialize(self, scheduler_address, cache_dir): :type cache_dir: str ''' self.scheduler_address = scheduler_address - while cache_dir.endswith('/'): - cache_dir = cache_dir[:-1] - self.cache_dir = cache_dir + '/run/' + self.cache_dir = cache_dir async def post(self): ''' @@ -63,9 +62,7 @@ async def post(self): ''' sim_request = SimulationRunRequest.parse(self.request.body) if sim_request.namespace is not None: - while sim_request.namespace.endswith('/'): - sim_request.namespace = sim_request.namespace[:-1] - self.cache_dir = '{sim_request.namespace}/{self.cache_dir}' + self.cache_dir = os.path.join(self.cache_dir, sim_request.namespace) self.key = sim_request.key cache = Cache(self.cache_dir, self.key) if cache.exists(): @@ -75,7 +72,7 @@ async def post(self): cache.create() client = Client(self.scheduler_address) future = self._submit(sim_request, client) - msg = '<{self.request.remote_ip}> | Simulation Run Request | <{self.key}> | Running simulation.' + msg = f'<{self.request.remote_ip}> | <{self.key}> | Running simulation.' log.info(msg) self._return_running() IOLoop.current().run_in_executor(None, self._cache, future, client) diff --git a/gillespy2/remote/server/simulation_run_cache.py b/gillespy2/remote/server/simulation_run_cache.py index ae7d77bd..d68ab88d 100644 --- a/gillespy2/remote/server/simulation_run_cache.py +++ b/gillespy2/remote/server/simulation_run_cache.py @@ -59,8 +59,11 @@ async def post(self): ''' sim_request = SimulationRunCacheRequest.parse(self.request.body) namespace = sim_request.namespace + log.debug('%(namespace)s', locals()) if namespace != '': - self.cache_dir = os.path.join(namespace, self.cache_dir) + namespaced_dir = os.path.join(namespace, self.cache_dir) + self.cache_dir = namespaced_dir + log.debug(namespaced_dir) sim_hash = sim_request.hash() cache = Cache(self.cache_dir, sim_hash) msg_0 = f'<{self.request.remote_ip}> | <{sim_hash}>' @@ -83,7 +86,7 @@ async def post(self): else: msg = f'{msg_0} | Returning cached results.' log.info(msg) - results = cache.get_sample() + results = cache.get_sample(n_traj) results_json = results.to_json() sim_response = SimulationRunCacheResponse(SimStatus.READY, results_id = sim_hash, results = results_json) self.write(sim_response.encode()) diff --git a/gillespy2/remote/server/status.py b/gillespy2/remote/server/status.py index 88d3f6e3..1ad8b993 100644 --- a/gillespy2/remote/server/status.py +++ b/gillespy2/remote/server/status.py @@ -1,7 +1,6 @@ ''' gillespy2.remote.server.status ''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. # This program is free software: you can redistribute it and/or modify @@ -20,7 +19,7 @@ from distributed import Client from tornado.web import RequestHandler from gillespy2.remote.core.errors import RemoteSimulationError -from gillespy2.remote.core.messages.status import SimStatus, StatusResponse +from gillespy2.remote.core.messages.status import SimStatus, StatusRequest, StatusResponse from gillespy2.remote.server.cache import Cache from gillespy2.remote.core.log_config import init_logging @@ -53,7 +52,7 @@ def initialize(self, scheduler_address, cache_dir): self.scheduler_address = scheduler_address self.cache_dir = cache_dir - async def get(self, results_id, n_traj, task_id): + async def post(self): ''' Process GET request. @@ -70,9 +69,12 @@ async def get(self, results_id, n_traj, task_id): self.set_status(404, reason=f'Malformed request: {self.request.uri}') self.finish() raise RemoteSimulationError(f'Malformed request: {self.request.uri}') + self.results_id = results_id - self.task_id = task_id n_traj = int(n_traj) + self.task_id = task_id + request = StatusRequest.parse(self.request.body) + log.debug(request) if results_id == task_id: while self.cache_dir.endswith('/'): From 432bf43112c1bf1f3ff624231284d4a9c5dbf445 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 29 May 2023 19:33:45 -0400 Subject: [PATCH 10/54] save --- gillespy2/remote/server/status.py | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/gillespy2/remote/server/status.py b/gillespy2/remote/server/status.py index 1ad8b993..b1d8ac45 100644 --- a/gillespy2/remote/server/status.py +++ b/gillespy2/remote/server/status.py @@ -54,17 +54,12 @@ def initialize(self, scheduler_address, cache_dir): async def post(self): ''' - Process GET request. + Process Status POST request. - :param results_id: Hash of the simulation. Required. - :type results_id: str - - :param n_traj: Number of trajectories in the request. Default 1. - :type n_traj: str - - :param task_id: ID of the running simulation. Required. - :type task_id: str ''' + request = StatusRequest.parse(self.request.body) + # TODO + if '' in (results_id, n_traj): self.set_status(404, reason=f'Malformed request: {self.request.uri}') self.finish() @@ -73,7 +68,6 @@ async def post(self): self.results_id = results_id n_traj = int(n_traj) self.task_id = task_id - request = StatusRequest.parse(self.request.body) log.debug(request) if results_id == task_id: From 3ea5baab13a556276d3817d4b9cdfefb654a3ed5 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 12 Jun 2023 17:47:57 -0400 Subject: [PATCH 11/54] quicksave --- .gitignore | 2 +- gillespy2/remote/cloud/ec2.py | 2 +- gillespy2/remote/core/exceptions.py | 6 +- .../remote/core/messages/simulation_run.py | 69 +++++++++--- gillespy2/remote/core/messages/status.py | 39 ++++--- gillespy2/remote/core/remote_results.py | 4 +- gillespy2/remote/core/remote_simulation.py | 6 +- gillespy2/remote/launch.py | 50 +++++---- gillespy2/remote/server/cache.py | 2 +- gillespy2/remote/server/results.py | 2 +- gillespy2/remote/server/simulation_run.py | 44 +++++--- gillespy2/remote/server/status.py | 102 ++++++++++-------- 12 files changed, 207 insertions(+), 121 deletions(-) diff --git a/.gitignore b/.gitignore index a42edd12..ab886c4b 100644 --- a/.gitignore +++ b/.gitignore @@ -70,4 +70,4 @@ __pycache__ htmlcov /build .idea - +env/ diff --git a/gillespy2/remote/cloud/ec2.py b/gillespy2/remote/cloud/ec2.py index 8e5b2719..650a8ae9 100644 --- a/gillespy2/remote/cloud/ec2.py +++ b/gillespy2/remote/cloud/ec2.py @@ -36,7 +36,7 @@ from paramiko import SSHClient, AutoAddPolicy except ImportError as err: name = __name__ - log.warn('boto3 and parkamiko are required for %(name)s') + log.warn('boto3 and parkamiko are required for %(name)s', locals()) def _ec2_logger(): diff --git a/gillespy2/remote/core/exceptions.py b/gillespy2/remote/core/exceptions.py index 1b4ea19a..71ccf8ad 100644 --- a/gillespy2/remote/core/exceptions.py +++ b/gillespy2/remote/core/exceptions.py @@ -21,4 +21,8 @@ class PRNGCollision(Exception): ''' ...Lucky??? ''' - \ No newline at end of file + +class MessageParseException(Exception): + ''' + Raised if there is an error parsing a raw HTTP message. + ''' \ No newline at end of file diff --git a/gillespy2/remote/core/messages/simulation_run.py b/gillespy2/remote/core/messages/simulation_run.py index fe2abb5a..3b4975e0 100644 --- a/gillespy2/remote/core/messages/simulation_run.py +++ b/gillespy2/remote/core/messages/simulation_run.py @@ -1,15 +1,19 @@ ''' gillespy2.remote.core.messages.simulation_run ''' +import copy +from hashlib import md5 +from json import JSONDecodeError from secrets import token_hex -from tornado.escape import json_decode +from tornado.escape import json_decode, json_encode from gillespy2 import Model +from gillespy2.remote.core.exceptions import MessageParseException from gillespy2.remote.core.messages.base import Request, Response from gillespy2.remote.core.messages.status import SimStatus class SimulationRunRequest(Request): ''' - A simulation request identifiable by a key. + A simulation request message object. :param model: A model to run. :type model: gillespy2.Model @@ -20,20 +24,26 @@ class SimulationRunRequest(Request): :param kwargs: kwargs for the model.run() call. :type kwargs: dict[str, Any] ''' - def __init__(self, model, namespace=None, **kwargs): + def __init__(self, model: Model, namespace=None, ignore_cache=False, **kwargs): self.model = model self.kwargs = kwargs - self.key = token_hex(16) + self.id = token_hex(32) + self.results_id = self.hash() self.namespace = namespace + self.ignore_cache = ignore_cache def encode(self): ''' JSON-encode model and then encode self to dict. ''' - return {'model': self.model.to_json(), + + return { + 'model': self.model.to_json(), 'kwargs': self.kwargs, - 'key': self.key, - 'namespace': self.namespace or '' + 'id': self.id, + 'results_id': self.results_id, + 'namespace': self.namespace, + 'ignore_cache': self.ignore_cache } @staticmethod @@ -45,15 +55,42 @@ def parse(raw_request): :type raw_request: dict[str, str] :returns: The decoded object. - :rtype: SimulationRunUniqueRequest + :rtype: SimulationRunRequest ''' - request_dict = json_decode(raw_request) - model = Model.from_json(request_dict['model']) - kwargs = request_dict['kwargs'] - namespace = request_dict['namespace'] - _ = SimulationRunRequest(model, namespace=namespace,**kwargs) - _.key = request_dict['key'] # apply correct token (from raw request) after object construction. + try: + request_dict = json_decode(raw_request) + except JSONDecodeError as err: + raise MessageParseException from err + + model = Model.from_json(request_dict.get('model', None)) + kwargs = request_dict.get('kwargs', {}) + id = request_dict.get('id', None) # apply correct token (from raw request) after object construction. + results_id = request_dict.get('results_id', None) # apply correct token (from raw request) after object construction. + namespace = request_dict.get('namespace', None) + ignore_cache = request_dict.get('ignore_cache', None) + if None in (model, id, results_id, ignore_cache): + raise MessageParseException + _ = SimulationRunRequest(model, namespace=namespace, ignore_cache=ignore_cache, **kwargs) + _.id = id # apply correct token (from raw request) after object construction. return _ + + def hash(self): + ''' + Generate a unique hash of this simulation request. + Does not include number_of_trajectories in this calculation. + + :returns: md5 hex digest. + :rtype: str + ''' + anon_model_string = self.model.to_anon().to_json(encode_private=False) + popped_kwargs = {kw:self.kwargs[kw] for kw in self.kwargs if kw!='number_of_trajectories'} + # Explanation of line above: + # Take 'self.kwargs' (a dict), and add all entries to a new dictionary, + # EXCEPT the 'number_of_trajectories' key/value pair. + kwargs_string = json_encode(popped_kwargs) + request_string = f'{anon_model_string}{kwargs_string}' + _hash = md5(str.encode(request_string)).hexdigest() + return _hash class SimulationRunResponse(Response): ''' @@ -74,9 +111,9 @@ def encode(self): ''' Encode self to dict. ''' + return {'status': self.status.name, - 'error_message': self.error_message or '', - } + 'error_message': self.error_message} @staticmethod def parse(raw_response): diff --git a/gillespy2/remote/core/messages/status.py b/gillespy2/remote/core/messages/status.py index ed52362e..8cb0f581 100644 --- a/gillespy2/remote/core/messages/status.py +++ b/gillespy2/remote/core/messages/status.py @@ -3,14 +3,14 @@ ''' from enum import Enum from tornado.escape import json_decode +from gillespy2.remote.core.exceptions import MessageParseException from gillespy2.remote.core.messages.base import Request, Response class SimStatus(Enum): ''' Status describing a remote simulation. ''' - PENDING = 'The simulation is pending.' - RUNNING = 'The simulation is still running.' + RUNNING = 'The simulation is currently running.' READY = 'Simulation is done and results exist in the cache.' ERROR = 'The Simulation has encountered an error.' DOES_NOT_EXIST = 'There is no evidence of this simulation either running or on disk.' @@ -20,8 +20,6 @@ def from_str(name): ''' Convert str to Enum. ''' - if name == 'PENDING': - return SimStatus.PENDING if name == 'RUNNING': return SimStatus.RUNNING if name == 'READY': @@ -38,11 +36,17 @@ class StatusRequest(Request): ''' A request for simulation status. - :param results_id: Hash of the SimulationRunRequest + :param results_id: Hash of the SimulationRunCacheRequest or key from SimulationRunRequest :type results_id: str + :param n_traj: Number of requested trajectories. Defaults to 1. + :type n_traj: int | None + + :param task_id: Handle to a currently running simulation. + :type task_id: str | None + :param namespace: Optional namespace to prepend to results directory. - :type namespace: str + :type namespace: str | None ''' def __init__(self, results_id, n_traj=None, task_id=None, namespace=None): self.results_id = results_id @@ -83,9 +87,9 @@ class StatusResponse(Response): :type status: SimStatus :param message: Possible error message or otherwise - :type message: str + :type message: str | None ''' - def __init__(self, status, message = None): + def __init__(self, status, message=None): self.status = status self.message = message @@ -93,10 +97,10 @@ def encode(self): ''' Encodes self. :returns: self as dict - :rtype: dict[str, str] + :rtype: dict ''' return {'status': self.status.name, - 'message': self.message or ''} + 'message': self.message} @staticmethod def parse(raw_response): @@ -109,10 +113,13 @@ def parse(raw_response): :returns: The decoded object. :rtype: StatusResponse ''' + response_dict = json_decode(raw_response) - status = SimStatus.from_str(response_dict['status']) - message = response_dict['message'] - if not message: - return StatusResponse(status) - else: - return StatusResponse(status, message) \ No newline at end of file + + try: + status = SimStatus.from_str(response_dict.get('status')) + except ValueError as err: + raise MessageParseException from err + + message = response_dict.get('message', None) + return StatusResponse(status, message=message) \ No newline at end of file diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py index 5e815af9..da2aea8a 100644 --- a/gillespy2/remote/core/remote_results.py +++ b/gillespy2/remote/core/remote_results.py @@ -113,8 +113,8 @@ def _status(self): raise Exception('TODO Name this exception class. Cant call status on a finished simulation.') # Request the status of a submitted simulation. status_request = StatusRequest(self.id, self.namespace) - response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, - f"/{self.id}/{self.n_traj}/{self.task_id}/status") + response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, + f"/status") if not response_raw.ok: raise RemoteSimulationError(response_raw.reason) diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index 78ae758d..d5177082 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -161,7 +161,7 @@ def _run_cache(self, request): return remote_results - def _run(self, request, ): + def _run(self, request): ''' Ignores the cache. Gives each simulation request a unique identifier. @@ -177,8 +177,8 @@ def _run(self, request, ): raise Exception(sim_response.error_message) # non-conforming object creation ... possible refactor needed to solve, so left in. remote_results = RemoteResults() - remote_results.id = request.key - remote_results.task_id = request.key + remote_results.id = request.id + remote_results.task_id = request.id remote_results.server = self.server remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) diff --git a/gillespy2/remote/launch.py b/gillespy2/remote/launch.py index 3929a669..8a5b8a42 100644 --- a/gillespy2/remote/launch.py +++ b/gillespy2/remote/launch.py @@ -22,6 +22,10 @@ from distributed import LocalCluster from gillespy2.remote.server.api import start_api +from logging import INFO, getLevelName +from gillespy2.remote.core.log_config import init_logging, set_global_log_level +log = init_logging(__name__) + def launch_server(): ''' Start the REST API. Alias to script "gillespy2-remote". @@ -32,17 +36,21 @@ def launch_server(): ''' def _parse_args() -> Namespace: desc = ''' - StochSS-Compute is a server and cache that anonymizes StochSS simulation data. - ''' +GillesPy2 Remote allows you to run simulations remotely on your own Dask cluster. +To launch both simultaneously, use `gillespy2-remote-cluster` instead. +Trajectories are automatically cached to support multiple users running the same model. +''' parser = ArgumentParser(description=desc, add_help=True, conflict_handler='resolve') server = parser.add_argument_group('Server') server.add_argument("-p", "--port", default=29681, type=int, required=False, help="The port to use for the server. Defaults to 29681.") + server.add_argument("-l", "--logging-level", default=INFO, required=False, + help="Set the logging level threshold. Str or int. Defaults to INFO (20)") cache = parser.add_argument_group('Cache') cache.add_argument('-c', '--cache_path', default='cache/', required=False, - help='Path to use for the cache. Default ./cache') + help='Path to use for the cache. Defaults to "cache/".') cache.add_argument('--rm', '--rm-cache', default=False, required=False, help='Whether to delete the cache upon exit. Default False.') @@ -54,12 +62,13 @@ def _parse_args() -> Namespace: return parser.parse_args() args = _parse_args() + asyncio.run(start_api(**args.__dict__)) def launch_with_cluster(): ''' - Start up a Dask Cluster and StochSS-Compute REST API. Alias to script "stochss-compute-cluster". + Start up a Dask cluster along with gillespy2.remote REST API. Alias to script "gillespy2-remote-cluster". `gillespy2-remote-cluster --help` OR @@ -67,20 +76,17 @@ def launch_with_cluster(): ''' def _parse_args() -> Namespace: - usage = ''' - gillespy2-remote-cluster -p PORT - ''' desc = ''' - Startup script for a StochSS-Compute cluster. - StochSS-Compute is a server and cache that anonymizes StochSS simulation data. - Uses Dask, a Python parallel computing library. - ''' - parser = ArgumentParser(description=desc, add_help=True, usage=usage, - conflict_handler='resolve') + Startup script for a GillesPy2 Remote and pre-configured Dask Distributed cluster. + Command-line options allow you to override automatic cluster configuration. + Your trajectories are automatically cached to support multiple users running the same model.''' + parser = ArgumentParser(description=desc, add_help=True, conflict_handler='resolve') server = parser.add_argument_group('Server') server.add_argument("-p", "--port", default=29681, type=int, required=False, help="The port to use for the server. Defaults to 29681.") + server.add_argument("-l", "--logging-level", default=INFO, required=False, + help='Set the logging level threshold. Str or int. Defaults to INFO (20).') cache = parser.add_argument_group('Cache') cache.add_argument('-c', '--cache_path', default='cache/', required=False, @@ -107,27 +113,31 @@ def _parse_args() -> Namespace: like ‘localhost:8787’ or ‘0.0.0.0:8787’. Defaults to ‘:8787’. \ Set to None to disable the dashboard. Use ‘:0’ for a random port.') dask.add_argument('-N', '--dask-name', default=None, required=False, - help='A name to use when printing out the cluster, defaults to type name.') + help='A name to use when printing out the cluster, defaults to the type name.') args = parser.parse_args() return args args = _parse_args() + args.logging_level = set_global_log_level(getLevelName(args.logging_level)) dask_args = {} for (arg, value) in vars(args).items(): if arg.startswith('dask_'): dask_args[arg[5:]] = value - print('Launching Dask Cluster...') + log.info('Launching Dask Cluster.') cluster = LocalCluster(**dask_args) tokens = cluster.scheduler_address.split(':') dask_host = tokens[1][2:] dask_port = int(tokens[2]) - print(f'Scheduler Address: <{cluster.scheduler_address}>') + msg = f'Scheduler Address: <{cluster.scheduler_address}>' + log.info(msg) for i, worker in cluster.workers.items(): - print(f'Worker {i}: {worker}') + msg = f'Worker {i}: {worker}' + log.info(msg) - print(f'Dashboard Link: <{cluster.dashboard_link}>\n') + msg = f'Dashboard Link: <{cluster.dashboard_link}>\n' + log.info(msg) try: asyncio.run(start_api(port=args.port, cache_path=args.cache_path, @@ -135,9 +145,9 @@ def _parse_args() -> Namespace: except asyncio.exceptions.CancelledError: pass finally: - print('Shutting down cluster...', end='') + log.info('Shutting down cluster.') asyncio.run(cluster.close()) - print('OK') + log.info('Cluster terminated.') if __name__ == '__main__': if len(sys.argv) > 1: diff --git a/gillespy2/remote/server/cache.py b/gillespy2/remote/server/cache.py index 6b0f8c6c..d067201b 100644 --- a/gillespy2/remote/server/cache.py +++ b/gillespy2/remote/server/cache.py @@ -35,9 +35,9 @@ class Cache: :type results_id: str ''' def __init__(self, cache_dir, results_id): - self.results_path = os.path.join(cache_dir, f'{results_id}.results') if not os.path.exists(cache_dir): os.makedirs(cache_dir) + self.results_path = os.path.join(cache_dir, f'{results_id}.results') def create(self) -> None: ''' diff --git a/gillespy2/remote/server/results.py b/gillespy2/remote/server/results.py index b03d7a90..17eb2f33 100644 --- a/gillespy2/remote/server/results.py +++ b/gillespy2/remote/server/results.py @@ -56,7 +56,7 @@ async def get(self, results_id = None): :param n_traj: Number of trajectories in the request. :type n_traj: str ''' - if results_id in ['', '/']: + if results_id in ('', '/'): self.set_status(404, reason=f'Malformed request: {self.request.uri}') self.finish() raise RemoteSimulationError(f'Malformed request | <{self.request.remote_ip}>') diff --git a/gillespy2/remote/server/simulation_run.py b/gillespy2/remote/server/simulation_run.py index c618fe2e..8e1979a6 100644 --- a/gillespy2/remote/server/simulation_run.py +++ b/gillespy2/remote/server/simulation_run.py @@ -63,24 +63,30 @@ async def post(self): sim_request = SimulationRunRequest.parse(self.request.body) if sim_request.namespace is not None: self.cache_dir = os.path.join(self.cache_dir, sim_request.namespace) - self.key = sim_request.key - cache = Cache(self.cache_dir, self.key) + cache = Cache(self.cache_dir, sim_request.id) if cache.exists(): - self.set_status(404, reason='Try again with a different key, because that one is taken.') + log.debug("This should not be happening.") self.finish() - raise PRNGCollision('Try again with a different key, because that one is taken.') - cache.create() - client = Client(self.scheduler_address) - future = self._submit(sim_request, client) - msg = f'<{self.request.remote_ip}> | <{self.key}> | Running simulation.' - log.info(msg) - self._return_running() - IOLoop.current().run_in_executor(None, self._cache, future, client) - - def _cache(self, future, client): + raise PRNGCollision + try: + cache.create() + client = Client(self.scheduler_address) + msg = f'<{self.request.remote_ip}> | <{sim_request.id}> | Running simulation.' + log.info(msg) + future = self._submit(sim_request, client) + IOLoop.current().run_in_executor(None, self._cache, cache, future, client) + self._return_running() + except Exception as err: + self._return_error(str(err)) + + + def _cache(self, cache, future, client): ''' Await results, close client, save to disk. + :param cache: Handle to the cache. + :type cache: Cache + :param future: Handle to the running simulation, to be awaited upon. :type future: distributed.Future @@ -89,7 +95,6 @@ def _cache(self, future, client): ''' results = future.result() client.close() - cache = Cache(self.cache_dir, self.key) cache.save(results) def _submit(self, sim_request, client): @@ -108,7 +113,8 @@ def _submit(self, sim_request, client): ''' model = sim_request.model kwargs = sim_request.kwargs - key = sim_request.key + key = sim_request.id + kwargs['seed'] = int(sim_request.id, 16) if "solver" in kwargs: # pylint:disable=import-outside-toplevel from pydoc import locate @@ -125,3 +131,11 @@ def _return_running(self): sim_response = SimulationRunResponse(SimStatus.RUNNING) self.write(sim_response.encode()) self.finish() + + def _return_error(self, error_message): + ''' + Let the user know we submitted the simulation to the scheduler. + ''' + sim_response = SimulationRunResponse(SimStatus.ERROR, error_message) + self.write(sim_response.encode()) + self.finish() diff --git a/gillespy2/remote/server/status.py b/gillespy2/remote/server/status.py index b1d8ac45..6f7a6f61 100644 --- a/gillespy2/remote/server/status.py +++ b/gillespy2/remote/server/status.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . +import os from distributed import Client from tornado.web import RequestHandler from gillespy2.remote.core.errors import RemoteSimulationError @@ -32,8 +33,6 @@ class StatusHandler(RequestHandler): def __init__(self, application, request, **kwargs): self.scheduler_address = None self.cache_dir = None - self.task_id = None - self.results_id = None super().__init__(application, request, **kwargs) def data_received(self, chunk: bytes): @@ -57,63 +56,78 @@ async def post(self): Process Status POST request. ''' - request = StatusRequest.parse(self.request.body) - # TODO - - if '' in (results_id, n_traj): - self.set_status(404, reason=f'Malformed request: {self.request.uri}') - self.finish() - raise RemoteSimulationError(f'Malformed request: {self.request.uri}') + status_request = StatusRequest.parse(self.request.body) + log.debug(status_request.__dict__) + + results_id = status_request.results_id + n_traj = int(status_request.n_traj) + task_id = status_request.task_id + namespace = status_request.namespace + + if namespace is not None: + self.cache_dir = os.path.join(self.cache_dir, namespace) + if results_id == task_id: # True iff call made using (ignore_cache=True) + self.cache_dir = os.path.join(self.cache_dir, 'run/') + + cache = Cache(self.cache_dir, results_id) - self.results_id = results_id - n_traj = int(n_traj) - self.task_id = task_id - log.debug(request) + msg_0 = f'<{self.request.remote_ip}> | Results ID: <{results_id}> | Trajectories: {n_traj} | Task ID: {task_id}' + log.info(msg_0) - if results_id == task_id: - while self.cache_dir.endswith('/'): - self.cache_dir = self.cache_dir[:-1] - self.cache_dir = self.cache_dir + '/run/' + msg_1 = f'<{results_id}> | <{task_id}> | Status:' + dne_msg = f'{msg_1} {SimStatus.DOES_NOT_EXIST.name}' + ready_msg = f'{msg_1} {SimStatus.READY.name}' + + if cache.exists(): + log.debug('cache.exists(): %(exists)s', locals()) + + if cache.is_empty(): + + if task_id is not None: + + state, err = await self._check_with_scheduler(task_id) - cache = Cache(self.cache_dir, results_id) - log_string = f'<{self.request.remote_ip}> | Results ID: <{results_id}> | Trajectories: {n_traj} | Task ID: {task_id}' - log.info(log_string) - - msg = f'<{results_id}> | <{task_id}> | Status: ' - - exists = cache.exists() - log.debug('exists: %(exists)s', locals()) - if exists: - empty = cache.is_empty() - if empty: - if self.task_id not in ('', None): - state, err = await self._check_with_scheduler() - log.info(msg + SimStatus.RUNNING.name + f' | Task: {state} | Error: {err}') + msg_2 = f'{msg_1} {SimStatus.RUNNING.name} | Task: {state} | Error: {err}' + log.info(msg_2) + if state == 'erred': self._respond_error(err) else: - self._respond_running(f'Scheduler task state: {state}') + self._respond_running(f'Scheduler Task State: {state}') + else: - log.info(msg+SimStatus.DOES_NOT_EXIST.name) + + log.info(dne_msg) self._respond_dne() + else: - ready = cache.is_ready(n_traj) - if ready: - log.info(msg+SimStatus.READY.name) + + if cache.is_ready(n_traj): + log.info(ready_msg) self._respond_ready() + else: - if self.task_id not in ('', None): - state, err = await self._check_with_scheduler() - log.info(msg+SimStatus.RUNNING.name+f' | Task: {state} | error: {err}') + + if task_id is not None: + + state, err = await self._check_with_scheduler(task_id) + + msg_2 = f'{msg_1} {SimStatus.RUNNING.name} | Task: {state} | Error: {err}' + log.info(msg_2) + if state == 'erred': self._respond_error(err) else: self._respond_running(f'Scheduler task state: {state}') + else: - log.info(msg+SimStatus.DOES_NOT_EXIST.name) + + log.info(dne_msg) self._respond_dne() + else: - log.info(msg+SimStatus.DOES_NOT_EXIST.name) + + log.info(dne_msg) self._respond_dne() def _respond_ready(self): @@ -136,7 +150,7 @@ def _respond_running(self, message): self.write(status_response.encode()) self.finish() - async def _check_with_scheduler(self): + async def _check_with_scheduler(self, task_id): ''' Ask the scheduler for information about a task. ''' @@ -152,6 +166,6 @@ def scheduler_task_state(task_id, dask_scheduler=None): return (task.state, task.exception_text) # Do not await. Reasons. It returns sync. - ret = client.run_on_scheduler(scheduler_task_state, self.task_id) + _ = client.run_on_scheduler(scheduler_task_state, task_id) client.close() - return ret + return _ From 7d09018299a0b5198320b0b325638291eb4fe34f Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 14 Jun 2023 14:30:15 -0400 Subject: [PATCH 12/54] consolidate --- gillespy2/remote/launch.py | 58 ++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 28 deletions(-) diff --git a/gillespy2/remote/launch.py b/gillespy2/remote/launch.py index 8a5b8a42..a328f6f3 100644 --- a/gillespy2/remote/launch.py +++ b/gillespy2/remote/launch.py @@ -23,9 +23,29 @@ from gillespy2.remote.server.api import start_api from logging import INFO, getLevelName -from gillespy2.remote.core.log_config import init_logging, set_global_log_level +from gillespy2.remote.core.log_config import init_logging log = init_logging(__name__) +def _add_shared_args(parser): + ''' + :type parser: ArgumentParser + + :rtype: ArgumentParser + ''' + + server = parser.add_argument_group('Server') + server.add_argument("-p", "--port", default=29681, type=int, required=False, + help="The port to use for the server. Defaults to 29681.") + server.add_argument("-l", "--logging-level", default=INFO, required=False, + help='Set the logging level threshold. Str or int. Defaults to INFO (20).') + + cache = parser.add_argument_group('Cache') + cache.add_argument('-c', '--cache_path', default='cache/', required=False, + help='Path to use for the cache.') + cache.add_argument('--rm', default=False, action='store_true', required=False, + help='Whether to delete the cache upon exit. Default False.') + + def launch_server(): ''' Start the REST API. Alias to script "gillespy2-remote". @@ -42,17 +62,7 @@ def _parse_args() -> Namespace: ''' parser = ArgumentParser(description=desc, add_help=True, conflict_handler='resolve') - server = parser.add_argument_group('Server') - server.add_argument("-p", "--port", default=29681, type=int, required=False, - help="The port to use for the server. Defaults to 29681.") - server.add_argument("-l", "--logging-level", default=INFO, required=False, - help="Set the logging level threshold. Str or int. Defaults to INFO (20)") - - cache = parser.add_argument_group('Cache') - cache.add_argument('-c', '--cache_path', default='cache/', required=False, - help='Path to use for the cache. Defaults to "cache/".') - cache.add_argument('--rm', '--rm-cache', default=False, required=False, - help='Whether to delete the cache upon exit. Default False.') + parser = _add_shared_args(parser) dask = parser.add_argument_group('Dask') dask.add_argument("-H", "--dask-host", default='localhost', required=False, @@ -62,6 +72,7 @@ def _parse_args() -> Namespace: return parser.parse_args() args = _parse_args() + args.logging_level = getLevelName(args.logging_level) asyncio.run(start_api(**args.__dict__)) @@ -82,17 +93,7 @@ def _parse_args() -> Namespace: Your trajectories are automatically cached to support multiple users running the same model.''' parser = ArgumentParser(description=desc, add_help=True, conflict_handler='resolve') - server = parser.add_argument_group('Server') - server.add_argument("-p", "--port", default=29681, type=int, required=False, - help="The port to use for the server. Defaults to 29681.") - server.add_argument("-l", "--logging-level", default=INFO, required=False, - help='Set the logging level threshold. Str or int. Defaults to INFO (20).') - - cache = parser.add_argument_group('Cache') - cache.add_argument('-c', '--cache_path', default='cache/', required=False, - help='Path to use for the cache.') - cache.add_argument('--rm', default=False, action='store_true', required=False, - help='Whether to delete the cache upon exit. Default False.') + parser = _add_shared_args(parser) dask = parser.add_argument_group('Dask') dask.add_argument("-H", "--dask-host", default=None, required=False, @@ -113,13 +114,13 @@ def _parse_args() -> Namespace: like ‘localhost:8787’ or ‘0.0.0.0:8787’. Defaults to ‘:8787’. \ Set to None to disable the dashboard. Use ‘:0’ for a random port.') dask.add_argument('-N', '--dask-name', default=None, required=False, - help='A name to use when printing out the cluster, defaults to the type name.') - args = parser.parse_args() - return args + help='A name to use when printing out the cluster, defaults to the type name.') + + return parser.parse_args() args = _parse_args() - args.logging_level = set_global_log_level(getLevelName(args.logging_level)) + args.logging_level = getLevelName(args.logging_level) dask_args = {} for (arg, value) in vars(args).items(): @@ -141,7 +142,7 @@ def _parse_args() -> Namespace: try: asyncio.run(start_api(port=args.port, cache_path=args.cache_path, - dask_host=dask_host, dask_scheduler_port=dask_port, rm=args.rm)) + dask_host=dask_host, dask_scheduler_port=dask_port, rm=args.rm, logging_level=args.logging_level)) except asyncio.exceptions.CancelledError: pass finally: @@ -149,6 +150,7 @@ def _parse_args() -> Namespace: asyncio.run(cluster.close()) log.info('Cluster terminated.') + if __name__ == '__main__': if len(sys.argv) > 1: if sys.argv[1] == 'cluster': From 7be4288233d9efab59fd6c7b787167c9bea8b148 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 14 Jun 2023 14:59:21 -0400 Subject: [PATCH 13/54] save --- .../remote/core/messages/simulation_run.py | 132 ---------------- .../core/messages/simulation_run_cache.py | 65 ++++++-- gillespy2/remote/core/remote_simulation.py | 21 ++- gillespy2/remote/launch.py | 2 + gillespy2/remote/server/api.py | 17 +-- gillespy2/remote/server/simulation_run.py | 141 ------------------ .../remote/server/simulation_run_cache.py | 21 +-- 7 files changed, 77 insertions(+), 322 deletions(-) delete mode 100644 gillespy2/remote/core/messages/simulation_run.py delete mode 100644 gillespy2/remote/server/simulation_run.py diff --git a/gillespy2/remote/core/messages/simulation_run.py b/gillespy2/remote/core/messages/simulation_run.py deleted file mode 100644 index 3b4975e0..00000000 --- a/gillespy2/remote/core/messages/simulation_run.py +++ /dev/null @@ -1,132 +0,0 @@ -''' -gillespy2.remote.core.messages.simulation_run -''' -import copy -from hashlib import md5 -from json import JSONDecodeError -from secrets import token_hex -from tornado.escape import json_decode, json_encode -from gillespy2 import Model -from gillespy2.remote.core.exceptions import MessageParseException -from gillespy2.remote.core.messages.base import Request, Response -from gillespy2.remote.core.messages.status import SimStatus - -class SimulationRunRequest(Request): - ''' - A simulation request message object. - - :param model: A model to run. - :type model: gillespy2.Model - - :param namespace: Optional namespace for the results. - :type namespace: str | None - - :param kwargs: kwargs for the model.run() call. - :type kwargs: dict[str, Any] - ''' - def __init__(self, model: Model, namespace=None, ignore_cache=False, **kwargs): - self.model = model - self.kwargs = kwargs - self.id = token_hex(32) - self.results_id = self.hash() - self.namespace = namespace - self.ignore_cache = ignore_cache - - def encode(self): - ''' - JSON-encode model and then encode self to dict. - ''' - - return { - 'model': self.model.to_json(), - 'kwargs': self.kwargs, - 'id': self.id, - 'results_id': self.results_id, - 'namespace': self.namespace, - 'ignore_cache': self.ignore_cache - } - - @staticmethod - def parse(raw_request): - ''' - Parse raw HTTP request. Done server-side. - - :param raw_request: The request. - :type raw_request: dict[str, str] - - :returns: The decoded object. - :rtype: SimulationRunRequest - ''' - try: - request_dict = json_decode(raw_request) - except JSONDecodeError as err: - raise MessageParseException from err - - model = Model.from_json(request_dict.get('model', None)) - kwargs = request_dict.get('kwargs', {}) - id = request_dict.get('id', None) # apply correct token (from raw request) after object construction. - results_id = request_dict.get('results_id', None) # apply correct token (from raw request) after object construction. - namespace = request_dict.get('namespace', None) - ignore_cache = request_dict.get('ignore_cache', None) - if None in (model, id, results_id, ignore_cache): - raise MessageParseException - _ = SimulationRunRequest(model, namespace=namespace, ignore_cache=ignore_cache, **kwargs) - _.id = id # apply correct token (from raw request) after object construction. - return _ - - def hash(self): - ''' - Generate a unique hash of this simulation request. - Does not include number_of_trajectories in this calculation. - - :returns: md5 hex digest. - :rtype: str - ''' - anon_model_string = self.model.to_anon().to_json(encode_private=False) - popped_kwargs = {kw:self.kwargs[kw] for kw in self.kwargs if kw!='number_of_trajectories'} - # Explanation of line above: - # Take 'self.kwargs' (a dict), and add all entries to a new dictionary, - # EXCEPT the 'number_of_trajectories' key/value pair. - kwargs_string = json_encode(popped_kwargs) - request_string = f'{anon_model_string}{kwargs_string}' - _hash = md5(str.encode(request_string)).hexdigest() - return _hash - -class SimulationRunResponse(Response): - ''' - A response from the server regarding a SimulationRunRequest. - - :param status: The status of the simulation. - :type status: SimStatus - - :param error_message: Possible error message. - :type error_message: str | None - - ''' - def __init__(self, status, error_message = None): - self.status = status - self.error_message = error_message - - def encode(self): - ''' - Encode self to dict. - ''' - - return {'status': self.status.name, - 'error_message': self.error_message} - - @staticmethod - def parse(raw_response): - ''' - Parse HTTP response. - - :param raw_response: The response. - :type raw_response: dict[str, str] - - :returns: The decoded object. - :rtype: SimulationRunResponse - ''' - response_dict = json_decode(raw_response) - status = SimStatus.from_str(response_dict['status']) - error_message = response_dict['error_message'] - return SimulationRunResponse(status, error_message) diff --git a/gillespy2/remote/core/messages/simulation_run_cache.py b/gillespy2/remote/core/messages/simulation_run_cache.py index a23216c5..17bccc32 100644 --- a/gillespy2/remote/core/messages/simulation_run_cache.py +++ b/gillespy2/remote/core/messages/simulation_run_cache.py @@ -2,35 +2,54 @@ gillespy2.remote.core.messages.simulation_run_cache ''' from hashlib import md5 +from json import JSONDecodeError +from secrets import token_hex from tornado.escape import json_decode, json_encode from gillespy2 import Results -from gillespy2.remote.core.messages.base import Response -from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest +from gillespy2.core.model import Model +from gillespy2.remote.core.exceptions import MessageParseException +from gillespy2.remote.core.messages.base import Request, Response from gillespy2.remote.core.messages.status import SimStatus -class SimulationRunCacheRequest(SimulationRunRequest): +class SimulationRunCacheRequest(Request): ''' - A simulation request. + A simulation request message object. :param model: A model to run. :type model: gillespy2.Model + :param namespace: Optional namespace for the results. + :type namespace: str | None + :param kwargs: kwargs for the model.run() call. :type kwargs: dict[str, Any] ''' - def __init__(self, model, namespace=None, **kwargs): - return super().__init__(model, namespace=namespace, **kwargs) + def __init__(self, model, namespace=None, ignore_cache=False, **kwargs): + self.model = model + self.kwargs = kwargs + self.id = token_hex(32) + self.results_id = self.hash() + self.namespace = namespace + self.ignore_cache = ignore_cache def encode(self): ''' JSON-encode model and then encode self to dict. ''' - return super().encode() + + return { + 'model': self.model.to_json(), + 'kwargs': self.kwargs, + 'id': self.id, + 'results_id': self.results_id, + 'namespace': self.namespace, + 'ignore_cache': self.ignore_cache + } @staticmethod def parse(raw_request): ''' - Parse HTTP request. + Parse raw HTTP request. Done server-side. :param raw_request: The request. :type raw_request: dict[str, str] @@ -38,10 +57,23 @@ def parse(raw_request): :returns: The decoded object. :rtype: SimulationRunRequest ''' - # return SimulationRunCacheRequest() - _ = SimulationRunRequest.parse(raw_request) - return SimulationRunCacheRequest(_.model, namespace=_.namespace, **_.kwargs) - + try: + request_dict = json_decode(raw_request) + except JSONDecodeError as err: + raise MessageParseException from err + + model = Model.from_json(request_dict.get('model', None)) + kwargs = request_dict.get('kwargs', {}) + id = request_dict.get('id', None) # apply correct token (from raw request) after object construction. + results_id = request_dict.get('results_id', None) # apply correct token (from raw request) after object construction. + namespace = request_dict.get('namespace', None) + ignore_cache = request_dict.get('ignore_cache', None) + if None in (model, id, results_id, ignore_cache): + raise MessageParseException + _ = SimulationRunCacheRequest(model, namespace=namespace, ignore_cache=ignore_cache, **kwargs) + _.id = id # apply correct token (from raw request) after object construction. + return _ + def hash(self): ''' Generate a unique hash of this simulation request. @@ -60,6 +92,7 @@ def hash(self): _hash = md5(str.encode(request_string)).hexdigest() return _hash + class SimulationRunCacheResponse(Response): ''' A response from the server regarding a SimulationRunCacheRequest. @@ -91,10 +124,10 @@ def encode(self): Encode self to dict. ''' return {'status': self.status.name, - 'error_message': self.error_message or '', - 'results_id': self.results_id or '', - 'results': self.results or '', - 'task_id': self.task_id or '',} + 'error_message': self.error_message, + 'results_id': self.results_id, + 'results': self.results, + 'task_id': self.task_id,} @staticmethod def parse(raw_response): diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index d5177082..6938084e 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -18,7 +18,6 @@ # along with this program. If not, see . from gillespy2.remote.client.endpoint import Endpoint -from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse from gillespy2.remote.core.messages.simulation_run_cache import SimulationRunCacheRequest, SimulationRunCacheResponse from gillespy2.remote.core.messages.status import SimStatus from gillespy2.remote.core.errors import RemoteSimulationError @@ -94,7 +93,7 @@ def is_cached(self, namespace=None, **params): sim_request = SimulationRunCacheRequest(model=self.model, namespace=namespace, **params) results_dummy = RemoteResults() - results_dummy.id = sim_request.hash() + results_dummy.id = sim_request.results_id results_dummy.server = self.server results_dummy.n_traj = params.get('number_of_trajectories', 1) return results_dummy.is_ready @@ -128,19 +127,16 @@ def run(self, namespace=None, ignore_cache=False, **params): params["solver"] = f"{params['solver'].__module__}.{params['solver'].__qualname__}" if self.solver is not None: params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" - if ignore_cache is True: - sim_request = SimulationRunRequest(self.model, namespace=namespace, **params) - return self._run(sim_request) - if ignore_cache is False: - sim_request = SimulationRunCacheRequest(self.model, namespace=namespace, **params) - return self._run_cache(sim_request) + sim_request = SimulationRunCacheRequest(self.model, namespace=namespace, ignore_cache=ignore_cache, **params) + return self._run_cache(sim_request) def _run_cache(self, request): ''' :param request: Request to send to the server. Contains Model and related arguments. :type request: SimulationRunRequest ''' - response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub="/run/cache", request=request) + response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub='/run/cache', request=request) + if not response_raw.ok: raise Exception(response_raw.reason) @@ -154,8 +150,11 @@ def _run_cache(self, request): else: remote_results = RemoteResults() - remote_results.id = sim_response.results_id - remote_results.task_id = sim_response.task_id + if request.ignore_cache is True: + remote_results.id = request.id + else: + remote_results.id = request.results_id + remote_results.task_id = request.id remote_results.server = self.server remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) diff --git a/gillespy2/remote/launch.py b/gillespy2/remote/launch.py index a328f6f3..e98315b4 100644 --- a/gillespy2/remote/launch.py +++ b/gillespy2/remote/launch.py @@ -44,6 +44,8 @@ def _add_shared_args(parser): help='Path to use for the cache.') cache.add_argument('--rm', default=False, action='store_true', required=False, help='Whether to delete the cache upon exit. Default False.') + + return parser def launch_server(): diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py index 0e1a4aab..10d539f9 100644 --- a/gillespy2/remote/server/api.py +++ b/gillespy2/remote/server/api.py @@ -22,7 +22,7 @@ import subprocess from logging import DEBUG, INFO from tornado.web import Application -from gillespy2.remote.server.simulation_run import SimulationRunHandler +from gillespy2.remote.server.is_cached import IsCachedHandler from gillespy2.remote.server.simulation_run_cache import SimulationRunCacheHandler from gillespy2.remote.server.sourceip import SourceIpHandler from gillespy2.remote.server.status import StatusHandler @@ -36,34 +36,27 @@ def _make_app(dask_host, dask_scheduler_port, cache): 'cache_dir': cache} cache_arg = {'cache_dir': cache} return Application([ - (r'/api/v2/simulation/gillespy2/run', - SimulationRunHandler, - args), (r'/api/v2/simulation/gillespy2/run/cache', SimulationRunCacheHandler, args), - (r'/api/v2/simulation/gillespy2/status', + (r'/api/v2/simulation/gillespy2/(?P[0-9a-fA-F][0-9a-fA-F]*?)/(?P[0-9a-fA-F][0-9a-fA-F]*?)/status', StatusHandler, args), (r'/api/v2/simulation/gillespy2/(?P[0-9a-zA-Z][0-9a-zA-Z]*?)/results', ResultsHandler, cache_arg), - (r'/api/v2/simulation/gillespy2/(?P.*?)/(?P[1-9]\d*?)/results', - ResultsHandler, - cache_arg), - # (r'/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached', - # IsCachedHandler, {'cache_dir': cache}), + (r'/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached', + IsCachedHandler, {'cache_dir': cache}), (r'/api/v2/cloud/sourceip', SourceIpHandler), ]) async def start_api( port = 29681, - cache_trajectories = True, cache_path = 'cache/', dask_host = 'localhost', dask_scheduler_port = 8786, rm = False, - logging_level = DEBUG, + logging_level = INFO, ): """ Start the REST API with the following arguments. diff --git a/gillespy2/remote/server/simulation_run.py b/gillespy2/remote/server/simulation_run.py deleted file mode 100644 index 8e1979a6..00000000 --- a/gillespy2/remote/server/simulation_run.py +++ /dev/null @@ -1,141 +0,0 @@ -''' -gillespy2.remote.server.run -''' -# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import os -from tornado.web import RequestHandler -from tornado.ioloop import IOLoop -from distributed import Client -from gillespy2.remote.core.messages.status import SimStatus -from gillespy2.remote.core.exceptions import PRNGCollision -from gillespy2.remote.core.messages.simulation_run import SimulationRunRequest, SimulationRunResponse -from gillespy2.remote.server.cache import Cache - -from gillespy2.remote.core.log_config import init_logging -log = init_logging(__name__) - -class SimulationRunHandler(RequestHandler): - ''' - Endpoint for running GillesPy2 simulations. - ''' - - def __init__(self, application, request, **kwargs): - self.scheduler_address = None - self.cache_dir = None - self.key = None - super().__init__(application, request, **kwargs) - - def data_received(self, chunk: bytes): - raise NotImplementedError() - - def initialize(self, scheduler_address, cache_dir): - ''' - Sets the address to the Dask scheduler and the cache directory. - Creates a new directory for one-off results files identifiable by token. - - :param scheduler_address: Scheduler address. - :type scheduler_address: str - - :param cache_dir: Path to the cache. - :type cache_dir: str - ''' - self.scheduler_address = scheduler_address - self.cache_dir = cache_dir - - async def post(self): - ''' - Process simulation run POST request. - ''' - sim_request = SimulationRunRequest.parse(self.request.body) - if sim_request.namespace is not None: - self.cache_dir = os.path.join(self.cache_dir, sim_request.namespace) - cache = Cache(self.cache_dir, sim_request.id) - if cache.exists(): - log.debug("This should not be happening.") - self.finish() - raise PRNGCollision - try: - cache.create() - client = Client(self.scheduler_address) - msg = f'<{self.request.remote_ip}> | <{sim_request.id}> | Running simulation.' - log.info(msg) - future = self._submit(sim_request, client) - IOLoop.current().run_in_executor(None, self._cache, cache, future, client) - self._return_running() - except Exception as err: - self._return_error(str(err)) - - - def _cache(self, cache, future, client): - ''' - Await results, close client, save to disk. - - :param cache: Handle to the cache. - :type cache: Cache - - :param future: Handle to the running simulation, to be awaited upon. - :type future: distributed.Future - - :param client: Client to the Dask scheduler. Closing here for good measure, not sure if strictly necessary. - :type client: distributed.Client - ''' - results = future.result() - client.close() - cache.save(results) - - def _submit(self, sim_request, client): - ''' - Submit request to dask scheduler. - Uses pydoc.locate to convert str to solver class name object. - - :param sim_request: The user's request for a simulation. - :type sim_request: SimulationRunRequest - - :param client: Client to the Dask scheduler. - :type client: distributed.Client - - :returns: Handle to the running simulation and the results on the worker. - :rtype: distributed.Future - ''' - model = sim_request.model - kwargs = sim_request.kwargs - key = sim_request.id - kwargs['seed'] = int(sim_request.id, 16) - if "solver" in kwargs: - # pylint:disable=import-outside-toplevel - from pydoc import locate - # pylint:enable=import-outside-toplevel - kwargs["solver"] = locate(kwargs["solver"]) - - future = client.submit(model.run, **kwargs, key=key) - return future - - def _return_running(self): - ''' - Let the user know we submitted the simulation to the scheduler. - ''' - sim_response = SimulationRunResponse(SimStatus.RUNNING) - self.write(sim_response.encode()) - self.finish() - - def _return_error(self, error_message): - ''' - Let the user know we submitted the simulation to the scheduler. - ''' - sim_response = SimulationRunResponse(SimStatus.ERROR, error_message) - self.write(sim_response.encode()) - self.finish() diff --git a/gillespy2/remote/server/simulation_run_cache.py b/gillespy2/remote/server/simulation_run_cache.py index d68ab88d..d7edd817 100644 --- a/gillespy2/remote/server/simulation_run_cache.py +++ b/gillespy2/remote/server/simulation_run_cache.py @@ -64,8 +64,12 @@ async def post(self): namespaced_dir = os.path.join(namespace, self.cache_dir) self.cache_dir = namespaced_dir log.debug(namespaced_dir) - sim_hash = sim_request.hash() - cache = Cache(self.cache_dir, sim_hash) + if sim_request.ignore_cache is True: + cache = Cache(self.cache_dir, sim_request.id) + else: + cache = Cache(self.cache_dir, sim_request.results_id) + + sim_hash = sim_request.results_id msg_0 = f'<{self.request.remote_ip}> | <{sim_hash}>' if not cache.exists(): cache.create() @@ -99,10 +103,10 @@ async def post(self): self._return_running(sim_hash, future.key) IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) - def _cache(self, sim_hash, future, client) -> None: + def _cache(self, results_id, future, client) -> None: ''' - :param sim_hash: Incoming request. - :type sim_hash: SimulationRunCacheRequest + :param results_id: Key to results. + :type results_id: str :param future: Future that completes to gillespy2.Results. :type future: distributed.Future @@ -113,7 +117,7 @@ def _cache(self, sim_hash, future, client) -> None: ''' results = future.result() client.close() - cache = Cache(self.cache_dir, sim_hash) + cache = Cache(self.cache_dir, results_id) cache.save(results) def _submit(self, sim_request, client): @@ -129,15 +133,12 @@ def _submit(self, sim_request, client): ''' model = sim_request.model kwargs = sim_request.kwargs - n_traj = kwargs.get('number_of_trajectories', 1) - sim_hash_tag = sim_request.hash()[:-8] - key = f'{sim_hash_tag}{n_traj}{token_hex(8)}' if "solver" in kwargs: from pydoc import locate kwargs["solver"] = locate(kwargs["solver"]) - return client.submit(model.run, **kwargs, key=key) + return client.submit(model.run, **kwargs, key=sim_request.id) def _return_running(self, results_id, task_id): sim_response = SimulationRunCacheResponse(SimStatus.RUNNING, results_id=results_id, task_id=task_id) From 4a45bfc8760acc63696ec746b22adab89023975a Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Tue, 18 Jul 2023 12:11:40 -0400 Subject: [PATCH 14/54] accept request object --- gillespy2/remote/client/server.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/gillespy2/remote/client/server.py b/gillespy2/remote/client/server.py index 34afa519..65cf4666 100644 --- a/gillespy2/remote/client/server.py +++ b/gillespy2/remote/client/server.py @@ -46,7 +46,7 @@ def address(self): ''' return NotImplemented - def get(self, endpoint: Endpoint, sub: str): + def get(self, endpoint: Endpoint, sub: str, request: Request = None): ''' Send a GET request to endpoint. @@ -56,6 +56,9 @@ def get(self, endpoint: Endpoint, sub: str): :param sub: Final part of url string. :type sub: str + :param request: An object that inherits from Request. + :type request: Request + :returns: The HTTP response. :rtype: requests.Response ''' @@ -64,7 +67,9 @@ def get(self, endpoint: Endpoint, sub: str): sec = 3 while n_try <= 3: try: - return requests.get(url, timeout=30) + if request is not None: + return requests.get( url, timeout=30, json=request.encode()) + return requests.get( url, timeout=30) except ConnectionError: print(f"Connection refused by server. Retrying in {sec} seconds....") From 818d2fb976e2e2e273e694a0254daa364912f11a Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Tue, 18 Jul 2023 20:22:24 -0400 Subject: [PATCH 15/54] cleanup --- gillespy2/remote/client/server.py | 2 +- .../core/messages/simulation_run_cache.py | 13 ++++---- gillespy2/remote/core/remote_results.py | 32 ++++++++++--------- gillespy2/remote/core/remote_simulation.py | 23 ------------- gillespy2/remote/server/api.py | 14 ++++---- .../remote/server/{ => handlers}/is_cached.py | 0 .../remote/server/{ => handlers}/results.py | 19 +++++++---- .../{ => handlers}/simulation_run_cache.py | 2 +- .../remote/server/{ => handlers}/sourceip.py | 0 .../remote/server/{ => handlers}/status.py | 24 +++++++------- 10 files changed, 57 insertions(+), 72 deletions(-) rename gillespy2/remote/server/{ => handlers}/is_cached.py (100%) rename gillespy2/remote/server/{ => handlers}/results.py (83%) rename gillespy2/remote/server/{ => handlers}/simulation_run_cache.py (99%) rename gillespy2/remote/server/{ => handlers}/sourceip.py (100%) rename gillespy2/remote/server/{ => handlers}/status.py (89%) diff --git a/gillespy2/remote/client/server.py b/gillespy2/remote/client/server.py index 65cf4666..f1a2966f 100644 --- a/gillespy2/remote/client/server.py +++ b/gillespy2/remote/client/server.py @@ -68,7 +68,7 @@ def get(self, endpoint: Endpoint, sub: str, request: Request = None): while n_try <= 3: try: if request is not None: - return requests.get( url, timeout=30, json=request.encode()) + return requests.get(url, timeout=30, params=request.encode()) return requests.get( url, timeout=30) except ConnectionError: diff --git a/gillespy2/remote/core/messages/simulation_run_cache.py b/gillespy2/remote/core/messages/simulation_run_cache.py index 17bccc32..406b7d62 100644 --- a/gillespy2/remote/core/messages/simulation_run_cache.py +++ b/gillespy2/remote/core/messages/simulation_run_cache.py @@ -142,11 +142,10 @@ def parse(raw_response): ''' response_dict = json_decode(raw_response) status = SimStatus.from_str(response_dict['status']) - results_id = response_dict['results_id'] - error_message = response_dict['error_message'] - task_id = response_dict['task_id'] - if response_dict['results'] != '': - results = Results.from_json(response_dict['results']) - else: - results = None + results_id = response_dict.get('results_id', None) + error_message = response_dict.get('error_message', None) + task_id = response_dict.get('task_id', None) + results = response_dict.get('results', None) + if results is not None: + results = Results.from_json(results) return SimulationRunCacheResponse(status, error_message, results_id, results, task_id) diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py index da2aea8a..8ecc7220 100644 --- a/gillespy2/remote/core/remote_results.py +++ b/gillespy2/remote/core/remote_results.py @@ -20,7 +20,7 @@ from gillespy2 import Results from gillespy2.remote.client.endpoint import Endpoint from gillespy2.remote.core.errors import RemoteSimulationError -from gillespy2.remote.core.messages.results import ResultsResponse +from gillespy2.remote.core.messages.results import ResultsRequest, ResultsResponse from gillespy2.remote.core.messages.status import StatusRequest, StatusResponse, SimStatus from gillespy2.remote.core.log_config import init_logging @@ -44,13 +44,16 @@ class RemoteResults(Results): :param task_id: Handle for the running simulation. :type task_id: str + + :param namespace: Optional namespace. + :type namespace: str ''' - # These four fields are initialized after object creation. - id = None + + id = None # required server = None - n_traj = None - task_id = None - namespace = None + n_traj = None # Defaults to 1 + task_id = None # optional + namespace = None # optional # pylint:disable=super-init-not-called def __init__(self, data = None): @@ -110,11 +113,12 @@ def _status(self): It is undefined/illegal behavior to call this function if self._data is not None. ''' if self._data is not None: - raise Exception('TODO Name this exception class. Cant call status on a finished simulation.') - # Request the status of a submitted simulation. - status_request = StatusRequest(self.id, self.namespace) - response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, - f"/status") + raise Exception('TODO Name this exception class. Cannot call status on a finished simulation.') + + status_request = StatusRequest(self.id, self.n_traj, self.task_id, self.namespace) + + response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, '/status', request=status_request) + if not response_raw.ok: raise RemoteSimulationError(response_raw.reason) @@ -142,10 +146,8 @@ def _resolve(self): if status == SimStatus.READY: log.info('Results ready. Fetching.......') - if self.id == self.task_id: - response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, f"/{self.id}/results") - else: - response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, f"/{self.id}/{self.n_traj}/results") + results_request = ResultsRequest(self.id, self.namespace) + response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, f"/results", request=results_request) if not response_raw.ok: raise RemoteSimulationError(response_raw.reason) diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index 6938084e..7179be4a 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -159,26 +159,3 @@ def _run_cache(self, request): remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) return remote_results - - def _run(self, request): - ''' - Ignores the cache. Gives each simulation request a unique identifier. - - :param request: Request to send to the server. Contains Model and related arguments. - :type request: SimulationRunUniqueRequest - ''' - response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub="/run", request=request) - - if not response_raw.ok: - raise Exception(response_raw.reason) - sim_response = SimulationRunResponse.parse(response_raw.text) - if not sim_response.status is SimStatus.RUNNING: - raise Exception(sim_response.error_message) - # non-conforming object creation ... possible refactor needed to solve, so left in. - remote_results = RemoteResults() - remote_results.id = request.id - remote_results.task_id = request.id - remote_results.server = self.server - remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) - - return remote_results diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py index 10d539f9..48015196 100644 --- a/gillespy2/remote/server/api.py +++ b/gillespy2/remote/server/api.py @@ -22,11 +22,11 @@ import subprocess from logging import DEBUG, INFO from tornado.web import Application -from gillespy2.remote.server.is_cached import IsCachedHandler -from gillespy2.remote.server.simulation_run_cache import SimulationRunCacheHandler -from gillespy2.remote.server.sourceip import SourceIpHandler -from gillespy2.remote.server.status import StatusHandler -from gillespy2.remote.server.results import ResultsHandler +from gillespy2.remote.server.handlers.is_cached import IsCachedHandler +from gillespy2.remote.server.handlers.simulation_run_cache import SimulationRunCacheHandler +from gillespy2.remote.server.handlers.sourceip import SourceIpHandler +from gillespy2.remote.server.handlers.status import StatusHandler +from gillespy2.remote.server.handlers.results import ResultsHandler from gillespy2.remote.core.log_config import init_logging, set_global_log_level log = init_logging(__name__) @@ -39,10 +39,10 @@ def _make_app(dask_host, dask_scheduler_port, cache): (r'/api/v2/simulation/gillespy2/run/cache', SimulationRunCacheHandler, args), - (r'/api/v2/simulation/gillespy2/(?P[0-9a-fA-F][0-9a-fA-F]*?)/(?P[0-9a-fA-F][0-9a-fA-F]*?)/status', + (r'/api/v2/simulation/gillespy2/status', StatusHandler, args), - (r'/api/v2/simulation/gillespy2/(?P[0-9a-zA-Z][0-9a-zA-Z]*?)/results', + (r'/api/v2/simulation/gillespy2/results', ResultsHandler, cache_arg), (r'/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached', diff --git a/gillespy2/remote/server/is_cached.py b/gillespy2/remote/server/handlers/is_cached.py similarity index 100% rename from gillespy2/remote/server/is_cached.py rename to gillespy2/remote/server/handlers/is_cached.py diff --git a/gillespy2/remote/server/results.py b/gillespy2/remote/server/handlers/results.py similarity index 83% rename from gillespy2/remote/server/results.py rename to gillespy2/remote/server/handlers/results.py index 17eb2f33..3eb1fc07 100644 --- a/gillespy2/remote/server/results.py +++ b/gillespy2/remote/server/handlers/results.py @@ -42,11 +42,11 @@ def initialize(self, cache_dir): :param cache_dir: Path to the cache. :type cache_dir: str ''' - while cache_dir.endswith('/'): - cache_dir = cache_dir[:-1] - self.cache_dir = cache_dir + '/run/' + # while cache_dir.endswith('/'): + # cache_dir = cache_dir[:-1] + self.cache_dir = cache_dir - async def get(self, results_id = None): + async def get(self): ''' Process GET request. @@ -56,15 +56,20 @@ async def get(self, results_id = None): :param n_traj: Number of trajectories in the request. :type n_traj: str ''' - if results_id in ('', '/'): + results_id = self.get_query_argument('results_id', None) + n_traj = self.get_query_argument('n_traj', 0) + if results_id is None: self.set_status(404, reason=f'Malformed request: {self.request.uri}') self.finish() raise RemoteSimulationError(f'Malformed request | <{self.request.remote_ip}>') msg = f' <{self.request.remote_ip}> | Results Request | <{results_id}>' log.info(msg) cache = Cache(self.cache_dir, results_id) - if cache.is_ready(): - results = cache.read() + if cache.is_ready(n_traj_wanted=n_traj): + if n_traj > 0: + results = cache.get_sample(n_traj) + else: + results = cache.read() results_response = ResultsResponse(results) self.write(results_response.encode()) else: diff --git a/gillespy2/remote/server/simulation_run_cache.py b/gillespy2/remote/server/handlers/simulation_run_cache.py similarity index 99% rename from gillespy2/remote/server/simulation_run_cache.py rename to gillespy2/remote/server/handlers/simulation_run_cache.py index d7edd817..c3184936 100644 --- a/gillespy2/remote/server/simulation_run_cache.py +++ b/gillespy2/remote/server/handlers/simulation_run_cache.py @@ -60,7 +60,7 @@ async def post(self): sim_request = SimulationRunCacheRequest.parse(self.request.body) namespace = sim_request.namespace log.debug('%(namespace)s', locals()) - if namespace != '': + if namespace is not None: namespaced_dir = os.path.join(namespace, self.cache_dir) self.cache_dir = namespaced_dir log.debug(namespaced_dir) diff --git a/gillespy2/remote/server/sourceip.py b/gillespy2/remote/server/handlers/sourceip.py similarity index 100% rename from gillespy2/remote/server/sourceip.py rename to gillespy2/remote/server/handlers/sourceip.py diff --git a/gillespy2/remote/server/status.py b/gillespy2/remote/server/handlers/status.py similarity index 89% rename from gillespy2/remote/server/status.py rename to gillespy2/remote/server/handlers/status.py index 6f7a6f61..1429f934 100644 --- a/gillespy2/remote/server/status.py +++ b/gillespy2/remote/server/handlers/status.py @@ -51,23 +51,25 @@ def initialize(self, scheduler_address, cache_dir): self.scheduler_address = scheduler_address self.cache_dir = cache_dir - async def post(self): + async def get(self): ''' - Process Status POST request. - + Process Status GET request. ''' - status_request = StatusRequest.parse(self.request.body) - log.debug(status_request.__dict__) + + # status_request = StatusRequest.parse(self.request) + log.debug(self.request.query_arguments) - results_id = status_request.results_id - n_traj = int(status_request.n_traj) - task_id = status_request.task_id - namespace = status_request.namespace + results_id = self.get_query_argument('results_id') + n_traj = self.get_query_argument('n_traj', None) + if n_traj is not None: + n_traj = int(n_traj) + task_id = self.get_query_argument('task_id', None) + namespace = self.get_query_argument('namespace', None) if namespace is not None: self.cache_dir = os.path.join(self.cache_dir, namespace) - if results_id == task_id: # True iff call made using (ignore_cache=True) - self.cache_dir = os.path.join(self.cache_dir, 'run/') + # if results_id == task_id: # True iff call made using (ignore_cache=True) + # self.cache_dir = os.path.join(self.cache_dir, 'run/') cache = Cache(self.cache_dir, results_id) From fab953a2f4f36b129a1ea8e099ad8fe1e53fde48 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 10:46:20 -0400 Subject: [PATCH 16/54] typo --- gillespy2/remote/launch.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gillespy2/remote/launch.py b/gillespy2/remote/launch.py index e98315b4..94f37592 100644 --- a/gillespy2/remote/launch.py +++ b/gillespy2/remote/launch.py @@ -52,9 +52,9 @@ def launch_server(): ''' Start the REST API. Alias to script "gillespy2-remote". - `gillespy2-remote --help` + `gillespy2-remote` OR - `python -m gillespy2.remote.launch --help` + `python -m gillespy2.remote.launch` ''' def _parse_args() -> Namespace: desc = ''' @@ -83,9 +83,9 @@ def launch_with_cluster(): ''' Start up a Dask cluster along with gillespy2.remote REST API. Alias to script "gillespy2-remote-cluster". - `gillespy2-remote-cluster --help` + `gillespy2-remote-cluster` OR - `python -m gillespy2.remote.launch cluster --help` + `python -m gillespy2.remote.launch cluster` ''' def _parse_args() -> Namespace: From 1b3286af9887ede3aec9bdb6ec91f7c40269f789 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 10:46:53 -0400 Subject: [PATCH 17/54] debug stuff --- gillespy2/remote/server/handlers/status.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gillespy2/remote/server/handlers/status.py b/gillespy2/remote/server/handlers/status.py index 1429f934..5c01f0e3 100644 --- a/gillespy2/remote/server/handlers/status.py +++ b/gillespy2/remote/server/handlers/status.py @@ -81,7 +81,7 @@ async def get(self): ready_msg = f'{msg_1} {SimStatus.READY.name}' if cache.exists(): - log.debug('cache.exists(): %(exists)s', locals()) + log.debug('cache.exists(): True') if cache.is_empty(): @@ -128,7 +128,7 @@ async def get(self): self._respond_dne() else: - + log.debug('cache.exists(): False') log.info(dne_msg) self._respond_dne() From 7278e93f524fa59070472b2dcd0f29aff9aee4af Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 10:47:48 -0400 Subject: [PATCH 18/54] cut out --- .../server/handlers/simulation_run_cache.py | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/gillespy2/remote/server/handlers/simulation_run_cache.py b/gillespy2/remote/server/handlers/simulation_run_cache.py index c3184936..2a2aaec8 100644 --- a/gillespy2/remote/server/handlers/simulation_run_cache.py +++ b/gillespy2/remote/server/handlers/simulation_run_cache.py @@ -58,19 +58,16 @@ async def post(self): Process simulation run request. ''' sim_request = SimulationRunCacheRequest.parse(self.request.body) + log.debug(sim_request.encode()) namespace = sim_request.namespace log.debug('%(namespace)s', locals()) if namespace is not None: namespaced_dir = os.path.join(namespace, self.cache_dir) self.cache_dir = namespaced_dir log.debug(namespaced_dir) - if sim_request.ignore_cache is True: - cache = Cache(self.cache_dir, sim_request.id) - else: - cache = Cache(self.cache_dir, sim_request.results_id) - - sim_hash = sim_request.results_id - msg_0 = f'<{self.request.remote_ip}> | <{sim_hash}>' + results_id = sim_request.results_id + cache = Cache(self.cache_dir, results_id) + msg_0 = f'<{self.request.remote_ip}> | <{results_id}>' if not cache.exists(): cache.create() empty = cache.is_empty() @@ -85,14 +82,14 @@ async def post(self): log.info(msg) client = Client(self.scheduler_address) future = self._submit(sim_request, client) - self._return_running(sim_hash, future.key) - IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) + self._return_running(results_id, future.key) + IOLoop.current().run_in_executor(None, self._cache, results_id, future, client) else: msg = f'{msg_0} | Returning cached results.' log.info(msg) results = cache.get_sample(n_traj) results_json = results.to_json() - sim_response = SimulationRunCacheResponse(SimStatus.READY, results_id = sim_hash, results = results_json) + sim_response = SimulationRunCacheResponse(SimStatus.READY, results_id = results_id, results = results_json) self.write(sim_response.encode()) self.finish() if empty: @@ -100,8 +97,8 @@ async def post(self): log.info(msg) client = Client(self.scheduler_address) future = self._submit(sim_request, client) - self._return_running(sim_hash, future.key) - IOLoop.current().run_in_executor(None, self._cache, sim_hash, future, client) + self._return_running(results_id, future.key) + IOLoop.current().run_in_executor(None, self._cache, results_id, future, client) def _cache(self, results_id, future, client) -> None: ''' From 65cbc6acb7ab35da65757d5581b93b74484bd510 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 10:49:58 -0400 Subject: [PATCH 19/54] handle it here! --- gillespy2/remote/core/messages/simulation_run_cache.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gillespy2/remote/core/messages/simulation_run_cache.py b/gillespy2/remote/core/messages/simulation_run_cache.py index 406b7d62..80175449 100644 --- a/gillespy2/remote/core/messages/simulation_run_cache.py +++ b/gillespy2/remote/core/messages/simulation_run_cache.py @@ -27,8 +27,11 @@ class SimulationRunCacheRequest(Request): def __init__(self, model, namespace=None, ignore_cache=False, **kwargs): self.model = model self.kwargs = kwargs - self.id = token_hex(32) - self.results_id = self.hash() + self.id = token_hex(16) + if ignore_cache is True: + self.results_id = self.id + else: + self.results_id = self.hash() self.namespace = namespace self.ignore_cache = ignore_cache @@ -72,6 +75,7 @@ def parse(raw_request): raise MessageParseException _ = SimulationRunCacheRequest(model, namespace=namespace, ignore_cache=ignore_cache, **kwargs) _.id = id # apply correct token (from raw request) after object construction. + _.results_id = results_id # apply correct token (from raw request) after object construction. return _ def hash(self): From ed66fd80fb3a31024bd11ad55e44e9a3ff7901db Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 10:50:15 -0400 Subject: [PATCH 20/54] still need to do is_cached --- gillespy2/remote/server/api.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py index 48015196..0337d9d3 100644 --- a/gillespy2/remote/server/api.py +++ b/gillespy2/remote/server/api.py @@ -36,18 +36,18 @@ def _make_app(dask_host, dask_scheduler_port, cache): 'cache_dir': cache} cache_arg = {'cache_dir': cache} return Application([ - (r'/api/v2/simulation/gillespy2/run/cache', + (r'/api/v3/simulation/gillespy2/run/cache', SimulationRunCacheHandler, args), - (r'/api/v2/simulation/gillespy2/status', + (r'/api/v3/simulation/gillespy2/status', StatusHandler, args), - (r'/api/v2/simulation/gillespy2/results', + (r'/api/v3/simulation/gillespy2/results', ResultsHandler, cache_arg), - (r'/api/v2/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached', + (r'/api/v3/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached', IsCachedHandler, {'cache_dir': cache}), - (r'/api/v2/cloud/sourceip', SourceIpHandler), + (r'/api/v3/cloud/sourceip', SourceIpHandler), ]) async def start_api( From 9cf79c7eb2b4d8c66c31dc652f43d77ae26b160a Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 10:50:25 -0400 Subject: [PATCH 21/54] typo --- gillespy2/remote/core/log_config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gillespy2/remote/core/log_config.py b/gillespy2/remote/core/log_config.py index 00b40b40..c9277207 100644 --- a/gillespy2/remote/core/log_config.py +++ b/gillespy2/remote/core/log_config.py @@ -13,8 +13,8 @@ def init_logging(name): Like so: - from gillespy2.remote.core.log_config import init_logs - log = init_logs(__name__) + from gillespy2.remote.core.log_config import init_logging + log = init_logging(__name__) :param name: Name for the logger. Use the dot-separated module path string. :type name: str From c3117579be38729eb5806482d77fb2bbbec7b313 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 10:51:24 -0400 Subject: [PATCH 22/54] moved check --- gillespy2/remote/core/remote_simulation.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index 7179be4a..ebb991b1 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -23,6 +23,9 @@ from gillespy2.remote.core.errors import RemoteSimulationError from gillespy2.remote.core.remote_results import RemoteResults +from gillespy2.remote.core.log_config import init_logging +log = init_logging(__name__) + class RemoteSimulation: ''' An object representing a remote gillespy2 simulation. Requires a model and a host address. @@ -150,10 +153,7 @@ def _run_cache(self, request): else: remote_results = RemoteResults() - if request.ignore_cache is True: - remote_results.id = request.id - else: - remote_results.id = request.results_id + remote_results.id = request.results_id remote_results.task_id = request.id remote_results.server = self.server remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) From e8b697e8a288f88534621849b22d59c4bbcbe74d Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 10:51:35 -0400 Subject: [PATCH 23/54] handy dev feature --- gillespy2/remote/core/debug.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 gillespy2/remote/core/debug.py diff --git a/gillespy2/remote/core/debug.py b/gillespy2/remote/core/debug.py new file mode 100644 index 00000000..f52d8893 --- /dev/null +++ b/gillespy2/remote/core/debug.py @@ -0,0 +1,23 @@ +''' +gillespy2.remote.core.debug +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from logging import getLogger +getLogger().setLevel('DEBUG') +# Call the line below in a python process to initiate client-side logging +# import gillespy2.remote.core.debug \ No newline at end of file From a56acdf438b365ecd2c0ada880925ee649c749b1 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 10:51:48 -0400 Subject: [PATCH 24/54] update version --- gillespy2/remote/client/server.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/gillespy2/remote/client/server.py b/gillespy2/remote/client/server.py index f1a2966f..49a6a9f1 100644 --- a/gillespy2/remote/client/server.py +++ b/gillespy2/remote/client/server.py @@ -23,6 +23,9 @@ from gillespy2.remote.client.endpoint import Endpoint from gillespy2.remote.core.messages.base import Request +from gillespy2.remote.core.log_config import init_logging +log = init_logging(__name__) + class Server(ABC): ''' Abstract Server class with hard coded endpoints. @@ -31,8 +34,8 @@ class Server(ABC): ''' _endpoints = { - Endpoint.SIMULATION_GILLESPY2: "/api/v2/simulation/gillespy2", - Endpoint.CLOUD: "/api/v2/cloud" + Endpoint.SIMULATION_GILLESPY2: "/api/v3/simulation/gillespy2", + Endpoint.CLOUD: "/api/v3/cloud" } def __init__(self) -> None: @@ -62,7 +65,9 @@ def get(self, endpoint: Endpoint, sub: str, request: Request = None): :returns: The HTTP response. :rtype: requests.Response ''' + log.debug(request.encode()) url = f"{self.address}{self._endpoints[endpoint]}{sub}" + log.debug(url) n_try = 1 sec = 3 while n_try <= 3: @@ -98,11 +103,13 @@ def post(self, endpoint: Endpoint, sub: str, request: Request = None): :returns: The HTTP response. :rtype: requests.Response ''' + log.debug(request.encode()) if self.address is NotImplemented: raise NotImplementedError url = f"{self.address}{self._endpoints[endpoint]}{sub}" + log.debug(url) n_try = 1 sec = 3 while n_try <= 3: From bfd0ffa0d77bc3340cd0a25de29faae832a50814 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 12:57:52 -0400 Subject: [PATCH 25/54] fix imports --- gillespy2/remote/client/server.py | 2 +- gillespy2/remote/cloud/ec2.py | 2 +- gillespy2/remote/core/remote_results.py | 2 +- gillespy2/remote/core/remote_simulation.py | 2 +- gillespy2/remote/core/{ => utils}/debug.py | 0 gillespy2/remote/core/{ => utils}/log_config.py | 4 ++-- gillespy2/remote/launch.py | 2 +- gillespy2/remote/server/api.py | 2 +- gillespy2/remote/server/handlers/results.py | 2 +- gillespy2/remote/server/handlers/simulation_run_cache.py | 2 +- gillespy2/remote/server/handlers/status.py | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) rename gillespy2/remote/core/{ => utils}/debug.py (100%) rename gillespy2/remote/core/{ => utils}/log_config.py (87%) diff --git a/gillespy2/remote/client/server.py b/gillespy2/remote/client/server.py index 49a6a9f1..11162cb5 100644 --- a/gillespy2/remote/client/server.py +++ b/gillespy2/remote/client/server.py @@ -23,7 +23,7 @@ from gillespy2.remote.client.endpoint import Endpoint from gillespy2.remote.core.messages.base import Request -from gillespy2.remote.core.log_config import init_logging +from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) class Server(ABC): diff --git a/gillespy2/remote/cloud/ec2.py b/gillespy2/remote/cloud/ec2.py index 650a8ae9..d240c438 100644 --- a/gillespy2/remote/cloud/ec2.py +++ b/gillespy2/remote/cloud/ec2.py @@ -26,7 +26,7 @@ from gillespy2.remote.core.messages.source_ip import SourceIpRequest, SourceIpResponse from gillespy2.remote.cloud.exceptions import EC2ImportException, ResourceException, EC2Exception from gillespy2.remote.client.endpoint import Endpoint -from gillespy2.remote.core.log_config import init_logging +from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) try: import boto3 diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py index 8ecc7220..2db4b350 100644 --- a/gillespy2/remote/core/remote_results.py +++ b/gillespy2/remote/core/remote_results.py @@ -23,7 +23,7 @@ from gillespy2.remote.core.messages.results import ResultsRequest, ResultsResponse from gillespy2.remote.core.messages.status import StatusRequest, StatusResponse, SimStatus -from gillespy2.remote.core.log_config import init_logging +from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) class RemoteResults(Results): diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index ebb991b1..0d099116 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -23,7 +23,7 @@ from gillespy2.remote.core.errors import RemoteSimulationError from gillespy2.remote.core.remote_results import RemoteResults -from gillespy2.remote.core.log_config import init_logging +from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) class RemoteSimulation: diff --git a/gillespy2/remote/core/debug.py b/gillespy2/remote/core/utils/debug.py similarity index 100% rename from gillespy2/remote/core/debug.py rename to gillespy2/remote/core/utils/debug.py diff --git a/gillespy2/remote/core/log_config.py b/gillespy2/remote/core/utils/log_config.py similarity index 87% rename from gillespy2/remote/core/log_config.py rename to gillespy2/remote/core/utils/log_config.py index c9277207..506d0243 100644 --- a/gillespy2/remote/core/log_config.py +++ b/gillespy2/remote/core/utils/log_config.py @@ -1,5 +1,5 @@ ''' -gillespy2.remote.core.log_config +gillespy2.remote.core.utils.log_config Global Logging Configuration ''' @@ -13,7 +13,7 @@ def init_logging(name): Like so: - from gillespy2.remote.core.log_config import init_logging + from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) :param name: Name for the logger. Use the dot-separated module path string. diff --git a/gillespy2/remote/launch.py b/gillespy2/remote/launch.py index 94f37592..9a58e989 100644 --- a/gillespy2/remote/launch.py +++ b/gillespy2/remote/launch.py @@ -23,7 +23,7 @@ from gillespy2.remote.server.api import start_api from logging import INFO, getLevelName -from gillespy2.remote.core.log_config import init_logging +from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) def _add_shared_args(parser): diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py index 0337d9d3..5da8b4eb 100644 --- a/gillespy2/remote/server/api.py +++ b/gillespy2/remote/server/api.py @@ -27,7 +27,7 @@ from gillespy2.remote.server.handlers.sourceip import SourceIpHandler from gillespy2.remote.server.handlers.status import StatusHandler from gillespy2.remote.server.handlers.results import ResultsHandler -from gillespy2.remote.core.log_config import init_logging, set_global_log_level +from gillespy2.remote.core.utils.log_config import init_logging, set_global_log_level log = init_logging(__name__) def _make_app(dask_host, dask_scheduler_port, cache): diff --git a/gillespy2/remote/server/handlers/results.py b/gillespy2/remote/server/handlers/results.py index 3eb1fc07..22c2c925 100644 --- a/gillespy2/remote/server/handlers/results.py +++ b/gillespy2/remote/server/handlers/results.py @@ -21,7 +21,7 @@ from gillespy2.remote.core.messages.results import ResultsResponse from gillespy2.remote.server.cache import Cache -from gillespy2.remote.core.log_config import init_logging +from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) class ResultsHandler(RequestHandler): diff --git a/gillespy2/remote/server/handlers/simulation_run_cache.py b/gillespy2/remote/server/handlers/simulation_run_cache.py index 2a2aaec8..966aae7d 100644 --- a/gillespy2/remote/server/handlers/simulation_run_cache.py +++ b/gillespy2/remote/server/handlers/simulation_run_cache.py @@ -26,7 +26,7 @@ from gillespy2.remote.core.messages.simulation_run_cache import SimulationRunCacheRequest, SimulationRunCacheResponse from gillespy2.remote.server.cache import Cache -from gillespy2.remote.core.log_config import init_logging +from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) class SimulationRunCacheHandler(RequestHandler): diff --git a/gillespy2/remote/server/handlers/status.py b/gillespy2/remote/server/handlers/status.py index 5c01f0e3..b5e6c182 100644 --- a/gillespy2/remote/server/handlers/status.py +++ b/gillespy2/remote/server/handlers/status.py @@ -23,7 +23,7 @@ from gillespy2.remote.core.messages.status import SimStatus, StatusRequest, StatusResponse from gillespy2.remote.server.cache import Cache -from gillespy2.remote.core.log_config import init_logging +from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) class StatusHandler(RequestHandler): From d3b8a3316fcd7d7e13737d442439e6ec856bd59b Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 14:28:49 -0400 Subject: [PATCH 26/54] adding parallelization --- .../server/handlers/simulation_run_cache.py | 77 +++++++++++++++++-- 1 file changed, 69 insertions(+), 8 deletions(-) diff --git a/gillespy2/remote/server/handlers/simulation_run_cache.py b/gillespy2/remote/server/handlers/simulation_run_cache.py index 966aae7d..7ebae7b3 100644 --- a/gillespy2/remote/server/handlers/simulation_run_cache.py +++ b/gillespy2/remote/server/handlers/simulation_run_cache.py @@ -17,7 +17,6 @@ # along with this program. If not, see . import os -from secrets import token_hex from tornado.web import RequestHandler from tornado.ioloop import IOLoop @@ -80,10 +79,7 @@ async def post(self): sim_request.kwargs['number_of_trajectories'] = trajectories_needed msg = f'{msg_0} | Partial cache. Running {trajectories_needed} new trajectories.' log.info(msg) - client = Client(self.scheduler_address) - future = self._submit(sim_request, client) - self._return_running(results_id, future.key) - IOLoop.current().run_in_executor(None, self._cache, results_id, future, client) + self._run_cache(sim_request) else: msg = f'{msg_0} | Returning cached results.' log.info(msg) @@ -95,11 +91,59 @@ async def post(self): if empty: msg = f'{msg_0} | Results not cached. Running simulation.' log.info(msg) - client = Client(self.scheduler_address) + self._run_cache(sim_request) + + def _run_cache(self, sim_request) -> None: + ''' + :param results_id: Key to results. + :type results_id: str + + :param future: Future that completes to gillespy2.Results. + :type future: distributed.Future + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + ''' + results_id = sim_request.results_id + client = Client(self.scheduler_address) + if sim_request.parallelize is True: + futures = self._submit_parallel(sim_request, client) + self._return_running(results_id, future.key) + + else: future = self._submit(sim_request, client) self._return_running(results_id, future.key) IOLoop.current().run_in_executor(None, self._cache, results_id, future, client) + async def _submit_parallel(sim_request, client): + ''' + :param sim_request: Incoming request. + :type sim_request: SimulationRunCacheRequest + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + :returns: Future that completes to gillespy2.Results. + :rtype: distributed.Future + ''' + model = sim_request.model + kwargs = sim_request.kwargs + + if "solver" in kwargs: + from pydoc import locate + kwargs["solver"] = locate(kwargs["solver"]) + + futures = [] + n_traj = kwargs['number_of_trajectories'] + kwargs['number_of_trajectories'] = 1 + for _ in range(n_traj): + future = client.submit(model.run, **kwargs) + futures.append(future) + + return futures + + def _cache(self, results_id, future, client) -> None: ''' :param results_id: Key to results. @@ -116,8 +160,25 @@ def _cache(self, results_id, future, client) -> None: client.close() cache = Cache(self.cache_dir, results_id) cache.save(results) + + def _cache(self, results_id, future, client) -> None: + ''' + :param results_id: Key to results. + :type results_id: str - def _submit(self, sim_request, client): + :param future: Future that completes to gillespy2.Results. + :type future: distributed.Future + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + ''' + results = future.result() + client.close() + cache = Cache(self.cache_dir, results_id) + cache.save(results) + + def _submit(self, sim_request, client: Client): ''' :param sim_request: Incoming request. :type sim_request: SimulationRunCacheRequest @@ -134,7 +195,7 @@ def _submit(self, sim_request, client): if "solver" in kwargs: from pydoc import locate kwargs["solver"] = locate(kwargs["solver"]) - + return client.submit(model.run, **kwargs, key=sim_request.id) def _return_running(self, results_id, task_id): From 7e501ed137e42a736d4d5f8df1055e862c6de6f2 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Wed, 19 Jul 2023 14:29:14 -0400 Subject: [PATCH 27/54] parallelization --- gillespy2/remote/core/messages/simulation_run_cache.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/gillespy2/remote/core/messages/simulation_run_cache.py b/gillespy2/remote/core/messages/simulation_run_cache.py index 80175449..b66d3f03 100644 --- a/gillespy2/remote/core/messages/simulation_run_cache.py +++ b/gillespy2/remote/core/messages/simulation_run_cache.py @@ -24,7 +24,7 @@ class SimulationRunCacheRequest(Request): :param kwargs: kwargs for the model.run() call. :type kwargs: dict[str, Any] ''' - def __init__(self, model, namespace=None, ignore_cache=False, **kwargs): + def __init__(self, model, namespace=None, ignore_cache=False, parallelize=False, **kwargs): self.model = model self.kwargs = kwargs self.id = token_hex(16) @@ -34,19 +34,20 @@ def __init__(self, model, namespace=None, ignore_cache=False, **kwargs): self.results_id = self.hash() self.namespace = namespace self.ignore_cache = ignore_cache + self.parallelize = parallelize def encode(self): ''' JSON-encode model and then encode self to dict. ''' - return { 'model': self.model.to_json(), 'kwargs': self.kwargs, 'id': self.id, 'results_id': self.results_id, 'namespace': self.namespace, - 'ignore_cache': self.ignore_cache + 'ignore_cache': self.ignore_cache, + 'parallelize': self.parallelize, } @staticmethod @@ -71,7 +72,8 @@ def parse(raw_request): results_id = request_dict.get('results_id', None) # apply correct token (from raw request) after object construction. namespace = request_dict.get('namespace', None) ignore_cache = request_dict.get('ignore_cache', None) - if None in (model, id, results_id, ignore_cache): + parallelize = request_dict.get('parallelize', None) + if None in (model, id, results_id, ignore_cache, parallelize): raise MessageParseException _ = SimulationRunCacheRequest(model, namespace=namespace, ignore_cache=ignore_cache, **kwargs) _.id = id # apply correct token (from raw request) after object construction. From 962973b10555a30068a0a6155ff4e3fad1c8e45d Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Thu, 20 Jul 2023 10:51:02 -0400 Subject: [PATCH 28/54] for erroneous status stuff --- gillespy2/remote/core/exceptions.py | 5 + mdip.demo.ipynb | 270 ++++++++++++++++++++++++++++ 2 files changed, 275 insertions(+) create mode 100644 mdip.demo.ipynb diff --git a/gillespy2/remote/core/exceptions.py b/gillespy2/remote/core/exceptions.py index 71ccf8ad..c228ad57 100644 --- a/gillespy2/remote/core/exceptions.py +++ b/gillespy2/remote/core/exceptions.py @@ -25,4 +25,9 @@ class PRNGCollision(Exception): class MessageParseException(Exception): ''' Raised if there is an error parsing a raw HTTP message. + ''' + +class StatusException(Exception): + ''' + Raised if there is an error determining the status of a simulation. ''' \ No newline at end of file diff --git a/mdip.demo.ipynb b/mdip.demo.ipynb new file mode 100644 index 00000000..6b163899 --- /dev/null +++ b/mdip.demo.ipynb @@ -0,0 +1,270 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-07-19 10:21:30,998 - gillespy2.remote.cloud.ec2 - WARNING - boto3 and parkamiko are required for gillespy2.remote.cloud.ec2\n" + ] + } + ], + "source": [ + "import gillespy2\n", + "import gillespy2.remote as remote\n", + "import gillespy2.remote.core.utils.debug " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def create_michaelis_menten(parameter_values=None):\n", + " # Intialize the Model with a name of your choosing.\n", + " model = gillespy2.Model(name=\"Michaelis_Menten\")\n", + "\n", + " \"\"\"\n", + " Variables (GillesPy2.Species) can be anything that participates in or is produced by a reaction channel.\n", + "\n", + " - name: A user defined name for the species.\n", + " - initial_value: A value/population count of species at start of simulation.\n", + " \"\"\"\n", + " A = gillespy2.Species(name=\"A\", initial_value=301)\n", + " B = gillespy2.Species(name=\"B\", initial_value=120)\n", + " C = gillespy2.Species(name=\"C\", initial_value=0)\n", + " D = gillespy2.Species(name=\"D\", initial_value=0)\n", + "\n", + " # Add the Variables to the Model.\n", + " model.add_species([A, B, C, D])\n", + "\n", + " \"\"\"\n", + " Parameters are constant values relevant to the system, such as reaction kinetic rates.\n", + "\n", + " - name: A user defined name for reference.\n", + " - expression: Some constant value.\n", + " \"\"\"\n", + " rate1 = gillespy2.Parameter(name=\"rate1\", expression=0.0017)\n", + " rate2 = gillespy2.Parameter(name=\"rate2\", expression=0.5)\n", + " rate3 = gillespy2.Parameter(name=\"rate3\", expression=0.1)\n", + "\n", + " # Add the Parameters to the Model.\n", + " model.add_parameter([rate1, rate2, rate3])\n", + "\n", + " \"\"\"\n", + " Reactions are the reaction channels which cause the system to change over time.\n", + "\n", + " - name: A user defined name for the reaction.\n", + " - reactants: A dictionary with participant reactants as keys, and consumed per reaction as value.\n", + " - products: A dictionary with reaction products as keys, and number formed per reaction as value.\n", + " - rate: A parameter rate constant to be applied to the propensity of this reaction firing.\n", + " - propensity_function: Can be used instead of rate in order to declare a custom propensity function in string format.\n", + " \"\"\"\n", + " r1 = gillespy2.Reaction(\n", + " name=\"r1\",\n", + " reactants={'A': 1, 'B': 1}, \n", + " products={'C': 1},\n", + " rate='rate1'\n", + " # propensity_function='1/0' # uncomment to cause error.\n", + " )\n", + "\n", + " r2 = gillespy2.Reaction(\n", + " name=\"r2\",\n", + " reactants={'C': 1}, \n", + " products={'A': 1, 'B': 1},\n", + " rate='rate2'\n", + " )\n", + "\n", + " r3 = gillespy2.Reaction(\n", + " name=\"r3\",\n", + " reactants={'C': 1}, \n", + " products={'B': 1, 'D': 1},\n", + " rate='rate3'\n", + " )\n", + "\n", + " # Add the Reactions to the Model.\n", + " model.add_reaction([r1, r2, r3])\n", + "\n", + " # Define the timespan of the model.\n", + " tspan = gillespy2.TimeSpan.linspace(t=100, num_points=100)\n", + " \n", + " # Set the timespan of the Model.\n", + " model.timespan(tspan)\n", + " return model" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "myModel = create_michaelis_menten()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "simulation = remote.RemoteSimulation(myModel, host='localhost', solver=gillespy2.NumPySSASolver)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-07-19 10:21:43,205 - gillespy2.remote.client.server - DEBUG - {'model': '{\\n \"_generate_translation_table\": true,\\n \"_hash_private_vars\": false,\\n \"_listOfAssignmentRules\": {},\\n \"_listOfEvents\": {},\\n \"_listOfFunctionDefinitions\": {},\\n \"_listOfParameters\": {\\n \"rate1\": \"P0\",\\n \"rate2\": \"P1\",\\n \"rate3\": \"P2\"\\n },\\n \"_listOfRateRules\": {},\\n \"_listOfReactions\": {\\n \"r1\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": null,\\n \"massaction\": false,\\n \"name\": \"R1\",\\n \"ode_propensity_function\": \"(((P0*S0)*S1)/vol)\",\\n \"products\": [\\n {\\n \"key\": \"S2\",\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(((P0*S0)*S1)/vol)\",\\n \"reactants\": [\\n {\\n \"key\": \"S0\",\\n \"value\": 1\\n },\\n {\\n \"key\": \"S1\",\\n \"value\": 1\\n }\\n ],\\n \"type\": \"customized\"\\n },\\n \"r2\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": null,\\n \"massaction\": false,\\n \"name\": \"R2\",\\n \"ode_propensity_function\": \"(P1*S2)\",\\n \"products\": [\\n {\\n \"key\": \"S0\",\\n \"value\": 1\\n },\\n {\\n \"key\": \"S1\",\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(P1*S2)\",\\n \"reactants\": [\\n {\\n \"key\": \"S2\",\\n \"value\": 1\\n }\\n ],\\n \"type\": \"customized\"\\n },\\n \"r3\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": null,\\n \"massaction\": false,\\n \"name\": \"R3\",\\n \"ode_propensity_function\": \"(P2*S2)\",\\n \"products\": [\\n {\\n \"key\": \"S1\",\\n \"value\": 1\\n },\\n {\\n \"key\": \"S3\",\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(P2*S2)\",\\n \"reactants\": [\\n {\\n \"key\": \"S2\",\\n \"value\": 1\\n }\\n ],\\n \"type\": \"customized\"\\n }\\n },\\n \"_listOfSpecies\": {\\n \"A\": \"S0\",\\n \"B\": \"S1\",\\n \"C\": \"S2\",\\n \"D\": \"S3\"\\n },\\n \"_type\": \"gillespy2.core.model.Model\",\\n \"annotation\": \"model\",\\n \"listOfAssignmentRules\": {},\\n \"listOfEvents\": {},\\n \"listOfFunctionDefinitions\": {},\\n \"listOfParameters\": {\\n \"rate1\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.0017\",\\n \"name\": \"rate1\",\\n \"value\": 0.0017\\n },\\n \"rate2\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.5\",\\n \"name\": \"rate2\",\\n \"value\": 0.5\\n },\\n \"rate3\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.1\",\\n \"name\": \"rate3\",\\n \"value\": 0.1\\n }\\n },\\n \"listOfRateRules\": {},\\n \"listOfReactions\": {\\n \"r1\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.0017\",\\n \"name\": \"rate1\",\\n \"value\": 0.0017\\n },\\n \"massaction\": true,\\n \"name\": \"r1\",\\n \"ode_propensity_function\": \"((rate1*A)*B)\",\\n \"products\": [\\n {\\n \"key\": {\\n \"_hash\": -5406967187597232989,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"C\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(((rate1*A)*B)/vol)\",\\n \"reactants\": [\\n {\\n \"key\": {\\n \"_hash\": 1689242874924442423,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 301,\\n \"mode\": null,\\n \"name\": \"A\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n },\\n {\\n \"key\": {\\n \"_hash\": 1522382677053280009,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 120,\\n \"mode\": null,\\n \"name\": \"B\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"type\": \"mass-action\"\\n },\\n \"r2\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.5\",\\n \"name\": \"rate2\",\\n \"value\": 0.5\\n },\\n \"massaction\": true,\\n \"name\": \"r2\",\\n \"ode_propensity_function\": \"(rate2*C)\",\\n \"products\": [\\n {\\n \"key\": {\\n \"_hash\": 1689242874924442423,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 301,\\n \"mode\": null,\\n \"name\": \"A\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n },\\n {\\n \"key\": {\\n \"_hash\": 1522382677053280009,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 120,\\n \"mode\": null,\\n \"name\": \"B\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(rate2*C)\",\\n \"reactants\": [\\n {\\n \"key\": {\\n \"_hash\": -5406967187597232989,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"C\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"type\": \"mass-action\"\\n },\\n \"r3\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.1\",\\n \"name\": \"rate3\",\\n \"value\": 0.1\\n },\\n \"massaction\": true,\\n \"name\": \"r3\",\\n \"ode_propensity_function\": \"(rate3*C)\",\\n \"products\": [\\n {\\n \"key\": {\\n \"_hash\": 1522382677053280009,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 120,\\n \"mode\": null,\\n \"name\": \"B\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n },\\n {\\n \"key\": {\\n \"_hash\": 8606284047498760059,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"D\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(rate3*C)\",\\n \"reactants\": [\\n {\\n \"key\": {\\n \"_hash\": -5406967187597232989,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"C\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"type\": \"mass-action\"\\n }\\n },\\n \"listOfSpecies\": {\\n \"A\": {\\n \"_hash\": 1689242874924442423,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 301,\\n \"mode\": null,\\n \"name\": \"A\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"B\": {\\n \"_hash\": 1522382677053280009,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 120,\\n \"mode\": null,\\n \"name\": \"B\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"C\": {\\n \"_hash\": -5406967187597232989,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"C\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"D\": {\\n \"_hash\": 8606284047498760059,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"D\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n }\\n },\\n \"name\": \"Michaelis_Menten\",\\n \"namespace\": {\\n \"rate1\": 0.0017,\\n \"rate2\": 0.5\\n },\\n \"tspan\": {\\n \"_type\": \"gillespy2.core.timespan.TimeSpan\",\\n \"items\": {\\n \"_type\": \"gillespy2.core.jsonify.NdArrayCoder\",\\n \"data\": [\\n 0.0,\\n 1.0101010101010102,\\n 2.0202020202020203,\\n 3.0303030303030303,\\n 4.040404040404041,\\n 5.050505050505051,\\n 6.0606060606060606,\\n 7.070707070707071,\\n 8.080808080808081,\\n 9.090909090909092,\\n 10.101010101010102,\\n 11.111111111111112,\\n 12.121212121212121,\\n 13.131313131313131,\\n 14.141414141414142,\\n 15.151515151515152,\\n 16.161616161616163,\\n 17.171717171717173,\\n 18.181818181818183,\\n 19.191919191919194,\\n 20.202020202020204,\\n 21.212121212121215,\\n 22.222222222222225,\\n 23.232323232323235,\\n 24.242424242424242,\\n 25.252525252525253,\\n 26.262626262626263,\\n 27.272727272727273,\\n 28.282828282828284,\\n 29.292929292929294,\\n 30.303030303030305,\\n 31.313131313131315,\\n 32.323232323232325,\\n 33.333333333333336,\\n 34.343434343434346,\\n 35.35353535353536,\\n 36.36363636363637,\\n 37.37373737373738,\\n 38.38383838383839,\\n 39.3939393939394,\\n 40.40404040404041,\\n 41.41414141414142,\\n 42.42424242424243,\\n 43.43434343434344,\\n 44.44444444444445,\\n 45.45454545454546,\\n 46.46464646464647,\\n 47.47474747474748,\\n 48.484848484848484,\\n 49.494949494949495,\\n 50.505050505050505,\\n 51.515151515151516,\\n 52.525252525252526,\\n 53.535353535353536,\\n 54.54545454545455,\\n 55.55555555555556,\\n 56.56565656565657,\\n 57.57575757575758,\\n 58.58585858585859,\\n 59.5959595959596,\\n 60.60606060606061,\\n 61.61616161616162,\\n 62.62626262626263,\\n 63.63636363636364,\\n 64.64646464646465,\\n 65.65656565656566,\\n 66.66666666666667,\\n 67.67676767676768,\\n 68.68686868686869,\\n 69.6969696969697,\\n 70.70707070707071,\\n 71.71717171717172,\\n 72.72727272727273,\\n 73.73737373737374,\\n 74.74747474747475,\\n 75.75757575757576,\\n 76.76767676767678,\\n 77.77777777777779,\\n 78.7878787878788,\\n 79.7979797979798,\\n 80.80808080808082,\\n 81.81818181818183,\\n 82.82828282828284,\\n 83.83838383838385,\\n 84.84848484848486,\\n 85.85858585858587,\\n 86.86868686868688,\\n 87.87878787878789,\\n 88.8888888888889,\\n 89.89898989898991,\\n 90.90909090909092,\\n 91.91919191919193,\\n 92.92929292929294,\\n 93.93939393939395,\\n 94.94949494949496,\\n 95.95959595959597,\\n 96.96969696969697,\\n 97.97979797979798,\\n 98.98989898989899,\\n 100.0\\n ]\\n }\\n },\\n \"units\": \"population\",\\n \"volume\": 1.0\\n}', 'kwargs': {'solver': 'gillespy2.solvers.numpy.ssa_solver.NumPySSASolver'}, 'id': 'a25f451380dc2ebe2049918d33ac6c67', 'results_id': 'a25f451380dc2ebe2049918d33ac6c67', 'namespace': None, 'ignore_cache': True}\n", + "2023-07-19 10:21:43,206 - gillespy2.remote.client.server - DEBUG - http://localhost:29681/api/v3/simulation/gillespy2/run/cache\n", + "2023-07-19 10:21:43,211 - urllib3.connectionpool - DEBUG - Starting new HTTP connection (1): localhost:29681\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[SimulationRunCacheRequest] http://localhost:29681/api/v3/simulation/gillespy2/run/cache\n", + "Unknown error: HTTPConnectionPool(host='localhost', port=29681): Max retries exceeded with url: /api/v3/simulation/gillespy2/run/cache (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')). Retrying in 3 seconds....\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-07-19 10:21:46,226 - urllib3.connectionpool - DEBUG - Starting new HTTP connection (1): localhost:29681\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[SimulationRunCacheRequest] http://localhost:29681/api/v3/simulation/gillespy2/run/cache\n", + "Unknown error: HTTPConnectionPool(host='localhost', port=29681): Max retries exceeded with url: /api/v3/simulation/gillespy2/run/cache (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')). Retrying in 6 seconds....\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-07-19 10:21:52,232 - urllib3.connectionpool - DEBUG - Starting new HTTP connection (1): localhost:29681\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[SimulationRunCacheRequest] http://localhost:29681/api/v3/simulation/gillespy2/run/cache\n", + "Unknown error: HTTPConnectionPool(host='localhost', port=29681): Max retries exceeded with url: /api/v3/simulation/gillespy2/run/cache (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')). Retrying in 18 seconds....\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mConnectionRefusedError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connection.py:174\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 173\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 174\u001b[0m conn \u001b[39m=\u001b[39m connection\u001b[39m.\u001b[39;49mcreate_connection(\n\u001b[1;32m 175\u001b[0m (\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_dns_host, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mport), \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtimeout, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mextra_kw\n\u001b[1;32m 176\u001b[0m )\n\u001b[1;32m 178\u001b[0m \u001b[39mexcept\u001b[39;00m SocketTimeout:\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/util/connection.py:95\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[39mif\u001b[39;00m err \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m---> 95\u001b[0m \u001b[39mraise\u001b[39;00m err\n\u001b[1;32m 97\u001b[0m \u001b[39mraise\u001b[39;00m socket\u001b[39m.\u001b[39merror(\u001b[39m\"\u001b[39m\u001b[39mgetaddrinfo returns an empty list\u001b[39m\u001b[39m\"\u001b[39m)\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m 84\u001b[0m sock\u001b[39m.\u001b[39mbind(source_address)\n\u001b[0;32m---> 85\u001b[0m sock\u001b[39m.\u001b[39;49mconnect(sa)\n\u001b[1;32m 86\u001b[0m \u001b[39mreturn\u001b[39;00m sock\n", + "\u001b[0;31mConnectionRefusedError\u001b[0m: [Errno 111] Connection refused", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mNewConnectionError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connectionpool.py:714\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m 713\u001b[0m \u001b[39m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 714\u001b[0m httplib_response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_make_request(\n\u001b[1;32m 715\u001b[0m conn,\n\u001b[1;32m 716\u001b[0m method,\n\u001b[1;32m 717\u001b[0m url,\n\u001b[1;32m 718\u001b[0m timeout\u001b[39m=\u001b[39;49mtimeout_obj,\n\u001b[1;32m 719\u001b[0m body\u001b[39m=\u001b[39;49mbody,\n\u001b[1;32m 720\u001b[0m headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[1;32m 721\u001b[0m chunked\u001b[39m=\u001b[39;49mchunked,\n\u001b[1;32m 722\u001b[0m )\n\u001b[1;32m 724\u001b[0m \u001b[39m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m 725\u001b[0m \u001b[39m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m 726\u001b[0m \u001b[39m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m 727\u001b[0m \u001b[39m# mess.\u001b[39;00m\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connectionpool.py:415\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m 414\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 415\u001b[0m conn\u001b[39m.\u001b[39;49mrequest(method, url, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mhttplib_request_kw)\n\u001b[1;32m 417\u001b[0m \u001b[39m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[1;32m 418\u001b[0m \u001b[39m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[1;32m 419\u001b[0m \u001b[39m# With this behaviour, the received response is still readable.\u001b[39;00m\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connection.py:244\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers)\u001b[0m\n\u001b[1;32m 243\u001b[0m headers[\u001b[39m\"\u001b[39m\u001b[39mUser-Agent\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m _get_default_user_agent()\n\u001b[0;32m--> 244\u001b[0m \u001b[39msuper\u001b[39;49m(HTTPConnection, \u001b[39mself\u001b[39;49m)\u001b[39m.\u001b[39;49mrequest(method, url, body\u001b[39m=\u001b[39;49mbody, headers\u001b[39m=\u001b[39;49mheaders)\n", + "File \u001b[0;32m~/.python/cpython.dip.build.3.11/lib/python3.11/http/client.py:1282\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m 1281\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Send a complete request to the server.\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1282\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_request(method, url, body, headers, encode_chunked)\n", + "File \u001b[0;32m~/.python/cpython.dip.build.3.11/lib/python3.11/http/client.py:1328\u001b[0m, in \u001b[0;36mHTTPConnection._send_request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m 1327\u001b[0m body \u001b[39m=\u001b[39m _encode(body, \u001b[39m'\u001b[39m\u001b[39mbody\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[0;32m-> 1328\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mendheaders(body, encode_chunked\u001b[39m=\u001b[39;49mencode_chunked)\n", + "File \u001b[0;32m~/.python/cpython.dip.build.3.11/lib/python3.11/http/client.py:1277\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1276\u001b[0m \u001b[39mraise\u001b[39;00m CannotSendHeader()\n\u001b[0;32m-> 1277\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_output(message_body, encode_chunked\u001b[39m=\u001b[39;49mencode_chunked)\n", + "File \u001b[0;32m~/.python/cpython.dip.build.3.11/lib/python3.11/http/client.py:1037\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1036\u001b[0m \u001b[39mdel\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_buffer[:]\n\u001b[0;32m-> 1037\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(msg)\n\u001b[1;32m 1039\u001b[0m \u001b[39mif\u001b[39;00m message_body \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 1040\u001b[0m \n\u001b[1;32m 1041\u001b[0m \u001b[39m# create a consistent interface to message_body\u001b[39;00m\n", + "File \u001b[0;32m~/.python/cpython.dip.build.3.11/lib/python3.11/http/client.py:975\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 974\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mauto_open:\n\u001b[0;32m--> 975\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconnect()\n\u001b[1;32m 976\u001b[0m \u001b[39melse\u001b[39;00m:\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connection.py:205\u001b[0m, in \u001b[0;36mHTTPConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mconnect\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m--> 205\u001b[0m conn \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_new_conn()\n\u001b[1;32m 206\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_prepare_conn(conn)\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connection.py:186\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 185\u001b[0m \u001b[39mexcept\u001b[39;00m SocketError \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m--> 186\u001b[0m \u001b[39mraise\u001b[39;00m NewConnectionError(\n\u001b[1;32m 187\u001b[0m \u001b[39mself\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mFailed to establish a new connection: \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m\"\u001b[39m \u001b[39m%\u001b[39m e\n\u001b[1;32m 188\u001b[0m )\n\u001b[1;32m 190\u001b[0m \u001b[39mreturn\u001b[39;00m conn\n", + "\u001b[0;31mNewConnectionError\u001b[0m: : Failed to establish a new connection: [Errno 111] Connection refused", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mMaxRetryError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/adapters.py:489\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 488\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m chunked:\n\u001b[0;32m--> 489\u001b[0m resp \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39;49murlopen(\n\u001b[1;32m 490\u001b[0m method\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mmethod,\n\u001b[1;32m 491\u001b[0m url\u001b[39m=\u001b[39;49murl,\n\u001b[1;32m 492\u001b[0m body\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mbody,\n\u001b[1;32m 493\u001b[0m headers\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mheaders,\n\u001b[1;32m 494\u001b[0m redirect\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 495\u001b[0m assert_same_host\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 496\u001b[0m preload_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 497\u001b[0m decode_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 498\u001b[0m retries\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_retries,\n\u001b[1;32m 499\u001b[0m timeout\u001b[39m=\u001b[39;49mtimeout,\n\u001b[1;32m 500\u001b[0m )\n\u001b[1;32m 502\u001b[0m \u001b[39m# Send the request.\u001b[39;00m\n\u001b[1;32m 503\u001b[0m \u001b[39melse\u001b[39;00m:\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connectionpool.py:798\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m 796\u001b[0m e \u001b[39m=\u001b[39m ProtocolError(\u001b[39m\"\u001b[39m\u001b[39mConnection aborted.\u001b[39m\u001b[39m\"\u001b[39m, e)\n\u001b[0;32m--> 798\u001b[0m retries \u001b[39m=\u001b[39m retries\u001b[39m.\u001b[39;49mincrement(\n\u001b[1;32m 799\u001b[0m method, url, error\u001b[39m=\u001b[39;49me, _pool\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m, _stacktrace\u001b[39m=\u001b[39;49msys\u001b[39m.\u001b[39;49mexc_info()[\u001b[39m2\u001b[39;49m]\n\u001b[1;32m 800\u001b[0m )\n\u001b[1;32m 801\u001b[0m retries\u001b[39m.\u001b[39msleep()\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/util/retry.py:592\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m 591\u001b[0m \u001b[39mif\u001b[39;00m new_retry\u001b[39m.\u001b[39mis_exhausted():\n\u001b[0;32m--> 592\u001b[0m \u001b[39mraise\u001b[39;00m MaxRetryError(_pool, url, error \u001b[39mor\u001b[39;00m ResponseError(cause))\n\u001b[1;32m 594\u001b[0m log\u001b[39m.\u001b[39mdebug(\u001b[39m\"\u001b[39m\u001b[39mIncremented Retry for (url=\u001b[39m\u001b[39m'\u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m): \u001b[39m\u001b[39m%r\u001b[39;00m\u001b[39m\"\u001b[39m, url, new_retry)\n", + "\u001b[0;31mMaxRetryError\u001b[0m: HTTPConnectionPool(host='localhost', port=29681): Max retries exceeded with url: /api/v3/simulation/gillespy2/run/cache (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mConnectionError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/GillesPy2/gillespy2/remote/client/server.py:121\u001b[0m, in \u001b[0;36mServer.post\u001b[0;34m(self, endpoint, sub, request)\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m[\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mtype\u001b[39m(request)\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m] \u001b[39m\u001b[39m{\u001b[39;00murl\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 121\u001b[0m \u001b[39mreturn\u001b[39;00m requests\u001b[39m.\u001b[39;49mpost(url, json\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mencode(), timeout\u001b[39m=\u001b[39;49m\u001b[39m30\u001b[39;49m)\n\u001b[1;32m 123\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mConnectionError\u001b[39;00m:\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/api.py:115\u001b[0m, in \u001b[0;36mpost\u001b[0;34m(url, data, json, **kwargs)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[39m\u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[1;32m 105\u001b[0m \n\u001b[1;32m 106\u001b[0m \u001b[39m:param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[39m:rtype: requests.Response\u001b[39;00m\n\u001b[1;32m 113\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 115\u001b[0m \u001b[39mreturn\u001b[39;00m request(\u001b[39m\"\u001b[39;49m\u001b[39mpost\u001b[39;49m\u001b[39m\"\u001b[39;49m, url, data\u001b[39m=\u001b[39;49mdata, json\u001b[39m=\u001b[39;49mjson, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[39mwith\u001b[39;00m sessions\u001b[39m.\u001b[39mSession() \u001b[39mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m \u001b[39mreturn\u001b[39;00m session\u001b[39m.\u001b[39;49mrequest(method\u001b[39m=\u001b[39;49mmethod, url\u001b[39m=\u001b[39;49murl, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/sessions.py:587\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 586\u001b[0m send_kwargs\u001b[39m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 587\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(prep, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49msend_kwargs)\n\u001b[1;32m 589\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/sessions.py:701\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[39m# Send the request\u001b[39;00m\n\u001b[0;32m--> 701\u001b[0m r \u001b[39m=\u001b[39m adapter\u001b[39m.\u001b[39;49msend(request, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 703\u001b[0m \u001b[39m# Total elapsed time of the request (approximately)\u001b[39;00m\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/adapters.py:565\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 563\u001b[0m \u001b[39mraise\u001b[39;00m SSLError(e, request\u001b[39m=\u001b[39mrequest)\n\u001b[0;32m--> 565\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mConnectionError\u001b[39;00m(e, request\u001b[39m=\u001b[39mrequest)\n\u001b[1;32m 567\u001b[0m \u001b[39mexcept\u001b[39;00m ClosedPoolError \u001b[39mas\u001b[39;00m e:\n", + "\u001b[0;31mConnectionError\u001b[0m: HTTPConnectionPool(host='localhost', port=29681): Max retries exceeded with url: /api/v3/simulation/gillespy2/run/cache (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m results \u001b[39m=\u001b[39m simulation\u001b[39m.\u001b[39;49mrun(ignore_cache\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n", + "File \u001b[0;32m~/GillesPy2/gillespy2/remote/core/remote_simulation.py:134\u001b[0m, in \u001b[0;36mRemoteSimulation.run\u001b[0;34m(self, namespace, ignore_cache, **params)\u001b[0m\n\u001b[1;32m 132\u001b[0m params[\u001b[39m\"\u001b[39m\u001b[39msolver\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msolver\u001b[39m.\u001b[39m\u001b[39m__module__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msolver\u001b[39m.\u001b[39m\u001b[39m__qualname__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m 133\u001b[0m sim_request \u001b[39m=\u001b[39m SimulationRunCacheRequest(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel, namespace\u001b[39m=\u001b[39mnamespace, ignore_cache\u001b[39m=\u001b[39mignore_cache, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mparams)\n\u001b[0;32m--> 134\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_run_cache(sim_request)\n", + "File \u001b[0;32m~/GillesPy2/gillespy2/remote/core/remote_simulation.py:141\u001b[0m, in \u001b[0;36mRemoteSimulation._run_cache\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 136\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_run_cache\u001b[39m(\u001b[39mself\u001b[39m, request):\n\u001b[1;32m 137\u001b[0m \u001b[39m \u001b[39m\u001b[39m'''\u001b[39;00m\n\u001b[1;32m 138\u001b[0m \u001b[39m :param request: Request to send to the server. Contains Model and related arguments.\u001b[39;00m\n\u001b[1;32m 139\u001b[0m \u001b[39m :type request: SimulationRunRequest\u001b[39;00m\n\u001b[1;32m 140\u001b[0m \u001b[39m '''\u001b[39;00m\n\u001b[0;32m--> 141\u001b[0m response_raw \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mserver\u001b[39m.\u001b[39;49mpost(Endpoint\u001b[39m.\u001b[39;49mSIMULATION_GILLESPY2, sub\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39m/run/cache\u001b[39;49m\u001b[39m'\u001b[39;49m, request\u001b[39m=\u001b[39;49mrequest)\n\u001b[1;32m 143\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m response_raw\u001b[39m.\u001b[39mok:\n\u001b[1;32m 144\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mException\u001b[39;00m(response_raw\u001b[39m.\u001b[39mreason)\n", + "File \u001b[0;32m~/GillesPy2/gillespy2/remote/client/server.py:130\u001b[0m, in \u001b[0;36mServer.post\u001b[0;34m(self, endpoint, sub, request)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n\u001b[1;32m 129\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mUnknown error: \u001b[39m\u001b[39m{\u001b[39;00merr\u001b[39m}\u001b[39;00m\u001b[39m. Retrying in \u001b[39m\u001b[39m{\u001b[39;00msec\u001b[39m}\u001b[39;00m\u001b[39m seconds....\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 130\u001b[0m sleep(sec)\n\u001b[1;32m 131\u001b[0m n_try \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n\u001b[1;32m 132\u001b[0m sec \u001b[39m*\u001b[39m\u001b[39m=\u001b[39m n_try\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "results = simulation.run(ignore_cache=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results.plot()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2+" + }, + "vscode": { + "interpreter": { + "hash": "ef56f9d787682a6ac61c29852cd545f557bd05dd1aa39793e08a6cd6ebb9b699" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 75ccc588ffbf2c045b63e7cb87e1adf25e912175 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Thu, 20 Jul 2023 10:51:29 -0400 Subject: [PATCH 29/54] handling ex. --- gillespy2/remote/core/remote_results.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py index 2db4b350..6982b2c7 100644 --- a/gillespy2/remote/core/remote_results.py +++ b/gillespy2/remote/core/remote_results.py @@ -20,6 +20,7 @@ from gillespy2 import Results from gillespy2.remote.client.endpoint import Endpoint from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.exceptions import StatusException from gillespy2.remote.core.messages.results import ResultsRequest, ResultsResponse from gillespy2.remote.core.messages.status import StatusRequest, StatusResponse, SimStatus @@ -83,8 +84,10 @@ def sim_status(self): :returns: Simulation status enum as a string. :rtype: str ''' - if self._data is not None: + if self._data is None: return self._status().status.name + else: + raise StatusException('Cannot call status on a finished simulation.') def get_gillespy2_results(self): """ @@ -113,7 +116,7 @@ def _status(self): It is undefined/illegal behavior to call this function if self._data is not None. ''' if self._data is not None: - raise Exception('TODO Name this exception class. Cannot call status on a finished simulation.') + raise StatusException('Cannot call status on a finished simulation.') status_request = StatusRequest(self.id, self.n_traj, self.task_id, self.namespace) From 4d7e0d2643b1f26f364b62acdaa3ea83d08567b2 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Thu, 20 Jul 2023 10:51:49 -0400 Subject: [PATCH 30/54] formatting --- gillespy2/remote/core/remote_simulation.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index 0d099116..6aef85fc 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -101,7 +101,7 @@ def is_cached(self, namespace=None, **params): results_dummy.n_traj = params.get('number_of_trajectories', 1) return results_dummy.is_ready - def run(self, namespace=None, ignore_cache=False, **params): + def run(self, namespace=None, ignore_cache=False, parallelize=False, submit_chunks=False, **params): # pylint:disable=line-too-long """ Simulate the Model on the target ComputeServer, returning the results or a handle to a running simulation. @@ -130,7 +130,12 @@ def run(self, namespace=None, ignore_cache=False, **params): params["solver"] = f"{params['solver'].__module__}.{params['solver'].__qualname__}" if self.solver is not None: params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" - sim_request = SimulationRunCacheRequest(self.model, namespace=namespace, ignore_cache=ignore_cache, **params) + sim_request = SimulationRunCacheRequest(self.model, + namespace=namespace, + ignore_cache=ignore_cache, + parallelize=parallelize, + submit_chunks=submit_chunks, + **params) return self._run_cache(sim_request) def _run_cache(self, request): @@ -159,3 +164,5 @@ def _run_cache(self, request): remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) return remote_results + +# def run_parallel(self, namespace=None): \ No newline at end of file From 31f1b47281014463a0e8d0aa2584f7c3c8dcd5b7 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Thu, 20 Jul 2023 10:52:27 -0400 Subject: [PATCH 31/54] new parameter --- .../remote/core/messages/simulation_run_cache.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/gillespy2/remote/core/messages/simulation_run_cache.py b/gillespy2/remote/core/messages/simulation_run_cache.py index b66d3f03..ae9b0664 100644 --- a/gillespy2/remote/core/messages/simulation_run_cache.py +++ b/gillespy2/remote/core/messages/simulation_run_cache.py @@ -24,7 +24,7 @@ class SimulationRunCacheRequest(Request): :param kwargs: kwargs for the model.run() call. :type kwargs: dict[str, Any] ''' - def __init__(self, model, namespace=None, ignore_cache=False, parallelize=False, **kwargs): + def __init__(self, model, namespace=None, ignore_cache=False, parallelize=False, chunk_trajectories=False, **kwargs): self.model = model self.kwargs = kwargs self.id = token_hex(16) @@ -35,6 +35,7 @@ def __init__(self, model, namespace=None, ignore_cache=False, parallelize=False, self.namespace = namespace self.ignore_cache = ignore_cache self.parallelize = parallelize + self.chunk_trajectories = chunk_trajectories def encode(self): ''' @@ -48,6 +49,7 @@ def encode(self): 'namespace': self.namespace, 'ignore_cache': self.ignore_cache, 'parallelize': self.parallelize, + 'chunk_trajectories': self.chunk_trajectories, } @staticmethod @@ -73,9 +75,15 @@ def parse(raw_request): namespace = request_dict.get('namespace', None) ignore_cache = request_dict.get('ignore_cache', None) parallelize = request_dict.get('parallelize', None) - if None in (model, id, results_id, ignore_cache, parallelize): + chunk_trajectories = request_dict.get('chunk_trajectories', None) + if None in (model, id, results_id, ignore_cache, parallelize, chunk_trajectories): raise MessageParseException - _ = SimulationRunCacheRequest(model, namespace=namespace, ignore_cache=ignore_cache, **kwargs) + _ = SimulationRunCacheRequest(model, + namespace=namespace, + ignore_cache=ignore_cache, + parallelize=parallelize, + chunk_trajectories=chunk_trajectories, + **kwargs) _.id = id # apply correct token (from raw request) after object construction. _.results_id = results_id # apply correct token (from raw request) after object construction. return _ From 1dca5f7e501adf8ca9793d6aa7f7784233c0b587 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Thu, 20 Jul 2023 10:52:49 -0400 Subject: [PATCH 32/54] put namespace logic here --- gillespy2/remote/server/cache.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/gillespy2/remote/server/cache.py b/gillespy2/remote/server/cache.py index d067201b..55c3b94d 100644 --- a/gillespy2/remote/server/cache.py +++ b/gillespy2/remote/server/cache.py @@ -24,6 +24,9 @@ from filelock import SoftFileLock from gillespy2 import Results +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + class Cache: ''' Cache @@ -34,7 +37,11 @@ class Cache: :param results_id: Simulation hash. :type results_id: str ''' - def __init__(self, cache_dir, results_id): + def __init__(self, cache_dir, results_id, namespace=None): + if namespace is not None: + namespaced_dir = os.path.join(cache_dir, namespace) + cache_dir = namespaced_dir + log.debug(namespaced_dir) if not os.path.exists(cache_dir): os.makedirs(cache_dir) self.results_path = os.path.join(cache_dir, f'{results_id}.results') @@ -167,17 +174,17 @@ def save(self, results: Results) -> None: :param results: The new Results. :type: gillespy2.Results ''' - msg = f'{datetime.now()} | Cache | <{self.results_path}> | ' + msg = f'<{self.results_path}> | ' lock = SoftFileLock(f'{self.results_path}.lock') with lock: with open(self.results_path, 'r+', encoding='utf-8') as file: try: old_results = Results.from_json(file.read()) combined_results = results + old_results - print(msg+'Add') + log.info(msg+'Add') file.seek(0) file.write(combined_results.to_json()) except JSONDecodeError: - print(msg+'New') + log.info(msg+'New') file.seek(0) file.write(results.to_json()) From 1a3ee3b3b4ac0a16cc1cfe4f9c2692d40fc943f5 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Thu, 20 Jul 2023 10:53:05 -0400 Subject: [PATCH 33/54] quicksave --- .../server/handlers/simulation_run_cache.py | 86 +++++++++++++++---- 1 file changed, 68 insertions(+), 18 deletions(-) diff --git a/gillespy2/remote/server/handlers/simulation_run_cache.py b/gillespy2/remote/server/handlers/simulation_run_cache.py index 7ebae7b3..78f87c1a 100644 --- a/gillespy2/remote/server/handlers/simulation_run_cache.py +++ b/gillespy2/remote/server/handlers/simulation_run_cache.py @@ -17,6 +17,7 @@ # along with this program. If not, see . import os +import random from tornado.web import RequestHandler from tornado.ioloop import IOLoop @@ -58,14 +59,10 @@ async def post(self): ''' sim_request = SimulationRunCacheRequest.parse(self.request.body) log.debug(sim_request.encode()) - namespace = sim_request.namespace log.debug('%(namespace)s', locals()) - if namespace is not None: - namespaced_dir = os.path.join(namespace, self.cache_dir) - self.cache_dir = namespaced_dir - log.debug(namespaced_dir) + results_id = sim_request.results_id - cache = Cache(self.cache_dir, results_id) + cache = Cache(self.cache_dir, results_id, namespace=sim_request.namespace) msg_0 = f'<{self.request.remote_ip}> | <{results_id}>' if not cache.exists(): cache.create() @@ -107,16 +104,19 @@ def _run_cache(self, sim_request) -> None: ''' results_id = sim_request.results_id client = Client(self.scheduler_address) + log.debug('_run_cache():') + log.debug(sim_request.parallelize) + log.debug(sim_request.chunk_trajectories) if sim_request.parallelize is True: - futures = self._submit_parallel(sim_request, client) - self._return_running(results_id, future.key) - + self._submit_parallel(sim_request, client) + elif sim_request.chunk_trajectories is True: + self._submit_chunks(sim_request, client) else: future = self._submit(sim_request, client) self._return_running(results_id, future.key) IOLoop.current().run_in_executor(None, self._cache, results_id, future, client) - async def _submit_parallel(sim_request, client): + def _submit_parallel(self, sim_request, client: Client): ''' :param sim_request: Incoming request. :type sim_request: SimulationRunCacheRequest @@ -129,35 +129,85 @@ async def _submit_parallel(sim_request, client): ''' model = sim_request.model kwargs = sim_request.kwargs - + results_id = sim_request.results_id if "solver" in kwargs: from pydoc import locate kwargs["solver"] = locate(kwargs["solver"]) futures = [] n_traj = kwargs['number_of_trajectories'] - kwargs['number_of_trajectories'] = 1 - for _ in range(n_traj): + del kwargs['number_of_trajectories'] + kwargs['seed'] = int(sim_request.id, 16) + for i in range(n_traj): + kwargs['seed'] += kwargs['seed'] future = client.submit(model.run, **kwargs) futures.append(future) + IOLoop.current().run_in_executor(None, self._cache_parallel, results_id, futures, client) + self._return_running(results_id, results_id) + + def _submit_chunks(self, sim_request, client: Client): + ''' + :param sim_request: Incoming request. + :type sim_request: SimulationRunCacheRequest + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + :returns: Future that completes to gillespy2.Results. + :rtype: distributed.Future + ''' + model = sim_request.model + kwargs = sim_request.kwargs + results_id = sim_request.results_id + if "solver" in kwargs: + from pydoc import locate + kwargs["solver"] = locate(kwargs["solver"]) + n_traj = kwargs.get('number_of_trajectories',1) + n_workers = len(client.scheduler_info()['workers']) + traj_per_worker = n_traj // n_workers + extra_traj = n_traj % n_workers + log.debug('_submit_chunks():') + log.debug(traj_per_worker) + log.debug(extra_traj) + # return + # traj_per_worker_list = [] + futures = [] + del kwargs['number_of_trajectories'] + kwargs['seed'] = int(sim_request.id, 16) + for _ in range(n_workers): + kwargs['seed'] += kwargs['seed'] + kwargs['number_of_trajectories'] = traj_per_worker + future = client.submit(model.run, **kwargs) + futures.append(future) + if extra_traj > 0: + kwargs['seed'] += kwargs['seed'] + kwargs['number_of_trajectories'] = extra_traj + future = client.submit(model.run, **kwargs) + futures.append(future) + + IOLoop.current().run_in_executor(None, self._cache_parallel, results_id, futures, client) + self._return_running(results_id, results_id) - return futures - def _cache(self, results_id, future, client) -> None: + def _cache_parallel(self, results_id, futures, client: Client) -> None: ''' :param results_id: Key to results. :type results_id: str - :param future: Future that completes to gillespy2.Results. - :type future: distributed.Future + :param future: List of Futures that completes to gillespy2.Results. + :type : List[distributed.Future] :param client: Handle to dask scheduler. :type client: distributed.Client ''' - results = future.result() + results = client.gather(futures, asynchronous=False) + results = sum(results) + log.debug('_cache_parallel():') + log.debug(results) client.close() + # return cache = Cache(self.cache_dir, results_id) cache.save(results) From 14af776c1e7942787df61c9088b7bd0e9e49307f Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Thu, 20 Jul 2023 10:53:31 -0400 Subject: [PATCH 34/54] namespace --- gillespy2/remote/server/handlers/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gillespy2/remote/server/handlers/status.py b/gillespy2/remote/server/handlers/status.py index b5e6c182..9d53fb66 100644 --- a/gillespy2/remote/server/handlers/status.py +++ b/gillespy2/remote/server/handlers/status.py @@ -71,7 +71,7 @@ async def get(self): # if results_id == task_id: # True iff call made using (ignore_cache=True) # self.cache_dir = os.path.join(self.cache_dir, 'run/') - cache = Cache(self.cache_dir, results_id) + cache = Cache(self.cache_dir, results_id, namespace=namespace) msg_0 = f'<{self.request.remote_ip}> | Results ID: <{results_id}> | Trajectories: {n_traj} | Task ID: {task_id}' log.info(msg_0) From 6d68058f2987b3338aebe78c7d2e370e2c326232 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Thu, 20 Jul 2023 11:58:16 -0400 Subject: [PATCH 35/54] finish namespace --- gillespy2/remote/core/messages/results.py | 2 +- gillespy2/remote/core/remote_simulation.py | 1 + gillespy2/remote/server/handlers/results.py | 3 ++- .../server/handlers/simulation_run_cache.py | 22 +++++++++---------- gillespy2/remote/server/handlers/status.py | 5 ----- 5 files changed, 15 insertions(+), 18 deletions(-) diff --git a/gillespy2/remote/core/messages/results.py b/gillespy2/remote/core/messages/results.py index 3c75e225..0f25dea7 100644 --- a/gillespy2/remote/core/messages/results.py +++ b/gillespy2/remote/core/messages/results.py @@ -40,7 +40,7 @@ def parse(raw_request): :rtype: ResultsRequest ''' request_dict = json_decode(raw_request) - return ResultsRequest(request_dict['results_id'], request_dict['results_id']) + return ResultsRequest(request_dict['results_id'], request_dict['namespace']) class ResultsResponse(Response): ''' diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index 6aef85fc..f15aef3d 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -162,6 +162,7 @@ def _run_cache(self, request): remote_results.task_id = request.id remote_results.server = self.server remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) + remote_results.namespace = request.namespace return remote_results diff --git a/gillespy2/remote/server/handlers/results.py b/gillespy2/remote/server/handlers/results.py index 22c2c925..767d69cf 100644 --- a/gillespy2/remote/server/handlers/results.py +++ b/gillespy2/remote/server/handlers/results.py @@ -58,13 +58,14 @@ async def get(self): ''' results_id = self.get_query_argument('results_id', None) n_traj = self.get_query_argument('n_traj', 0) + namespace = self.get_query_argument('namespace', None) if results_id is None: self.set_status(404, reason=f'Malformed request: {self.request.uri}') self.finish() raise RemoteSimulationError(f'Malformed request | <{self.request.remote_ip}>') msg = f' <{self.request.remote_ip}> | Results Request | <{results_id}>' log.info(msg) - cache = Cache(self.cache_dir, results_id) + cache = Cache(self.cache_dir, results_id, namespace=namespace) if cache.is_ready(n_traj_wanted=n_traj): if n_traj > 0: results = cache.get_sample(n_traj) diff --git a/gillespy2/remote/server/handlers/simulation_run_cache.py b/gillespy2/remote/server/handlers/simulation_run_cache.py index 78f87c1a..df2b2e32 100644 --- a/gillespy2/remote/server/handlers/simulation_run_cache.py +++ b/gillespy2/remote/server/handlers/simulation_run_cache.py @@ -62,25 +62,25 @@ async def post(self): log.debug('%(namespace)s', locals()) results_id = sim_request.results_id - cache = Cache(self.cache_dir, results_id, namespace=sim_request.namespace) + self.cache = Cache(self.cache_dir, results_id, namespace=sim_request.namespace) msg_0 = f'<{self.request.remote_ip}> | <{results_id}>' - if not cache.exists(): - cache.create() - empty = cache.is_empty() + if not self.cache.exists(): + self.cache.create() + empty = self.cache.is_empty() if not empty: # Check the number of trajectories in the request, default 1 n_traj = sim_request.kwargs.get('number_of_trajectories', 1) # Compare that to the number of cached trajectories - trajectories_needed = cache.n_traj_needed(n_traj) + trajectories_needed = self.cache.n_traj_needed(n_traj) if trajectories_needed > 0: sim_request.kwargs['number_of_trajectories'] = trajectories_needed - msg = f'{msg_0} | Partial cache. Running {trajectories_needed} new trajectories.' + msg = f'{msg_0} | Partial self.. Running {trajectories_needed} new trajectories.' log.info(msg) self._run_cache(sim_request) else: msg = f'{msg_0} | Returning cached results.' log.info(msg) - results = cache.get_sample(n_traj) + results = self.cache.get_sample(n_traj) results_json = results.to_json() sim_response = SimulationRunCacheResponse(SimStatus.READY, results_id = results_id, results = results_json) self.write(sim_response.encode()) @@ -208,8 +208,8 @@ def _cache_parallel(self, results_id, futures, client: Client) -> None: log.debug(results) client.close() # return - cache = Cache(self.cache_dir, results_id) - cache.save(results) + # cache = Cache(self.cache_dir, results_id) + self.cache.save(results) def _cache(self, results_id, future, client) -> None: ''' @@ -225,8 +225,8 @@ def _cache(self, results_id, future, client) -> None: ''' results = future.result() client.close() - cache = Cache(self.cache_dir, results_id) - cache.save(results) + # cache = Cache(self.cache_dir, results_id) + self.cache.save(results) def _submit(self, sim_request, client: Client): ''' diff --git a/gillespy2/remote/server/handlers/status.py b/gillespy2/remote/server/handlers/status.py index 9d53fb66..8a0ea9c7 100644 --- a/gillespy2/remote/server/handlers/status.py +++ b/gillespy2/remote/server/handlers/status.py @@ -66,11 +66,6 @@ async def get(self): task_id = self.get_query_argument('task_id', None) namespace = self.get_query_argument('namespace', None) - if namespace is not None: - self.cache_dir = os.path.join(self.cache_dir, namespace) - # if results_id == task_id: # True iff call made using (ignore_cache=True) - # self.cache_dir = os.path.join(self.cache_dir, 'run/') - cache = Cache(self.cache_dir, results_id, namespace=namespace) msg_0 = f'<{self.request.remote_ip}> | Results ID: <{results_id}> | Trajectories: {n_traj} | Task ID: {task_id}' From e4a3cadd513dabc841d8b6957935fe329655b06c Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 12:58:44 -0400 Subject: [PATCH 36/54] unneeded? --- gillespy2/remote/__version__.py | 76 ++++++++++++++++----------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/gillespy2/remote/__version__.py b/gillespy2/remote/__version__.py index 27cec9a8..a351a250 100644 --- a/gillespy2/remote/__version__.py +++ b/gillespy2/remote/__version__.py @@ -1,38 +1,38 @@ -''' -gillespy2.remote.__version__ -''' -# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. -# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. - -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. - -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. - -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - - -# ============================================================================= -# @file __version__.py -# @brief stochss-compute version info -# @license Please see the file named LICENSE.md in parent directory -# @website https://github.com/StochSS/stochss-compute -# ============================================================================= - - -__version__ = '1.0.1' - -__title__ = "stochss-compute" -__description__ = "A compute delegation package for the StochSS family of stochastic simulators" -__url__ = "https://github.com/StochSS/stochss-compute" -__download_url__ = "https://pypi.org/project/stochss-compute/#files" -__author__ = "See CONTRIBUTORS.md" -__email__ = "briandrawert@gmail.com" -__license__ = "GPLv3" -__copyright__ = "Copyright (C) 2017-2023" +# ''' +# gillespy2.remote.__version__ +# ''' +# # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# # This program is free software: you can redistribute it and/or modify +# # it under the terms of the GNU General Public License as published by +# # the Free Software Foundation, either version 3 of the License, or +# # (at your option) any later version. + +# # This program is distributed in the hope that it will be useful, +# # but WITHOUT ANY WARRANTY; without even the implied warranty of +# # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# # GNU General Public License for more details. + +# # You should have received a copy of the GNU General Public License +# # along with this program. If not, see . + + +# # ============================================================================= +# # @file __version__.py +# # @brief stochss-compute version info +# # @license Please see the file named LICENSE.md in parent directory +# # @website https://github.com/StochSS/stochss-compute +# # ============================================================================= + + +# __version__ = '1.0.1' + +# __title__ = "stochss-compute" +# __description__ = "A compute delegation package for the StochSS family of stochastic simulators" +# __url__ = "https://github.com/StochSS/stochss-compute" +# __download_url__ = "https://pypi.org/project/stochss-compute/#files" +# __author__ = "See CONTRIBUTORS.md" +# __email__ = "briandrawert@gmail.com" +# __license__ = "GPLv3" +# __copyright__ = "Copyright (C) 2017-2023" From 7724e4fe4befdce7f8dd4f8b5331be1fdccd230a Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 12:59:01 -0400 Subject: [PATCH 37/54] new option --- gillespy2/remote/launch.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/gillespy2/remote/launch.py b/gillespy2/remote/launch.py index 9a58e989..6887ee22 100644 --- a/gillespy2/remote/launch.py +++ b/gillespy2/remote/launch.py @@ -18,11 +18,11 @@ import sys import asyncio +from logging import INFO, getLevelName from argparse import ArgumentParser, Namespace from distributed import LocalCluster from gillespy2.remote.server.api import start_api - -from logging import INFO, getLevelName +from gillespy2.remote.core.utils.dask import silence_dask_logs from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) @@ -58,10 +58,10 @@ def launch_server(): ''' def _parse_args() -> Namespace: desc = ''' -GillesPy2 Remote allows you to run simulations remotely on your own Dask cluster. -To launch both simultaneously, use `gillespy2-remote-cluster` instead. -Trajectories are automatically cached to support multiple users running the same model. -''' + GillesPy2 Remote allows you to run simulations remotely on your own Dask cluster. + To launch both simultaneously, use `gillespy2-remote-cluster` instead. + Trajectories are automatically cached to support multiple users running the same model. + ''' parser = ArgumentParser(description=desc, add_help=True, conflict_handler='resolve') parser = _add_shared_args(parser) @@ -92,7 +92,8 @@ def _parse_args() -> Namespace: desc = ''' Startup script for a GillesPy2 Remote and pre-configured Dask Distributed cluster. Command-line options allow you to override automatic cluster configuration. - Your trajectories are automatically cached to support multiple users running the same model.''' + Your trajectories are automatically cached to support multiple users running the same model. + ''' parser = ArgumentParser(description=desc, add_help=True, conflict_handler='resolve') parser = _add_shared_args(parser) @@ -110,20 +111,23 @@ def _parse_args() -> Namespace: Default will let Dask decide based on your CPU.') dask.add_argument('--dask-processes', default=None, required=False, type=bool, help='Whether to use processes (True) or threads (False). \ - Defaults to True, unless worker_class=Worker, in which case it defaults to False.') + Defaults to True, unless worker_class=Worker, in which case it defaults to False.') # Keep as non-flag b/c of side effects. dask.add_argument('-D', '--dask-dashboard-address', default=':8787', required=False, help='Address on which to listen for the Bokeh diagnostics server \ like ‘localhost:8787’ or ‘0.0.0.0:8787’. Defaults to ‘:8787’. \ Set to None to disable the dashboard. Use ‘:0’ for a random port.') dask.add_argument('-N', '--dask-name', default=None, required=False, help='A name to use when printing out the cluster, defaults to the type name.') + dask.add_argument('--silence-dask', default=False, action='store_true', required=False, + help='Enable this flag to silence dask INFO log output.') return parser.parse_args() args = _parse_args() args.logging_level = getLevelName(args.logging_level) - + if args.silence_dask is True: + silence_dask_logs() dask_args = {} for (arg, value) in vars(args).items(): if arg.startswith('dask_'): From 5ca07e9a079c5cf930484f85d8df5e3312128a59 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 12:59:10 -0400 Subject: [PATCH 38/54] add endpoints --- gillespy2/remote/client/endpoint.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/gillespy2/remote/client/endpoint.py b/gillespy2/remote/client/endpoint.py index 680a780a..be8cdf7c 100644 --- a/gillespy2/remote/client/endpoint.py +++ b/gillespy2/remote/client/endpoint.py @@ -25,4 +25,6 @@ class Endpoint(Enum): ''' SIMULATION_GILLESPY2 = 1 CLOUD = 2 + CACHE = 3 + DASK = 4 \ No newline at end of file From fee16524b3fdb14e50892f7005bf401298df0d0b Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 12:59:19 -0400 Subject: [PATCH 39/54] new endpoints --- gillespy2/remote/client/server.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/gillespy2/remote/client/server.py b/gillespy2/remote/client/server.py index 11162cb5..99021ba7 100644 --- a/gillespy2/remote/client/server.py +++ b/gillespy2/remote/client/server.py @@ -35,7 +35,9 @@ class Server(ABC): _endpoints = { Endpoint.SIMULATION_GILLESPY2: "/api/v3/simulation/gillespy2", - Endpoint.CLOUD: "/api/v3/cloud" + Endpoint.CLOUD: "/api/v3/cloud", + Endpoint.CACHE: "/api/v3/cache", + Endpoint.DASK: '/api/v3/dask', } def __init__(self) -> None: @@ -65,7 +67,8 @@ def get(self, endpoint: Endpoint, sub: str, request: Request = None): :returns: The HTTP response. :rtype: requests.Response ''' - log.debug(request.encode()) + if request is not None: + log.debug(request.encode()) url = f"{self.address}{self._endpoints[endpoint]}{sub}" log.debug(url) n_try = 1 From 41cad70104e2ef8456fb2ec875b0753d465762fc Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 12:59:35 -0400 Subject: [PATCH 40/54] cleanup --- gillespy2/remote/cloud/ec2.py | 79 ++++++++++++++--------------------- 1 file changed, 31 insertions(+), 48 deletions(-) diff --git a/gillespy2/remote/cloud/ec2.py b/gillespy2/remote/cloud/ec2.py index d240c438..247c1e33 100644 --- a/gillespy2/remote/cloud/ec2.py +++ b/gillespy2/remote/cloud/ec2.py @@ -24,7 +24,7 @@ from gillespy2.remote.client.server import Server from gillespy2.remote.cloud.ec2_config import EC2LocalConfig, EC2RemoteConfig from gillespy2.remote.core.messages.source_ip import SourceIpRequest, SourceIpResponse -from gillespy2.remote.cloud.exceptions import EC2ImportException, ResourceException, EC2Exception +from gillespy2.remote.cloud.exceptions import ResourceException, EC2Exception from gillespy2.remote.client.endpoint import Endpoint from gillespy2.remote.core.utils.log_config import init_logging log = init_logging(__name__) @@ -36,23 +36,7 @@ from paramiko import SSHClient, AutoAddPolicy except ImportError as err: name = __name__ - log.warn('boto3 and parkamiko are required for %(name)s', locals()) - - -def _ec2_logger(): - log = logging.getLogger("EC2Cluster") - log.setLevel(logging.INFO) - log.propagate = False - - if not log.handlers: - _formatter = logging.Formatter( - '%(asctime)s - %(name)s - %(levelname)s - %(message)s') - _handler = logging.StreamHandler() - _handler.setFormatter(_formatter) - log.addHandler(_handler) - - return log - + log.warn('boto3 and paramiko are required for %(name)s', locals()) class EC2Cluster(Server): """ @@ -66,7 +50,6 @@ class EC2Cluster(Server): :raises EC2Exception: possible boto3 ClientError from AWS calls. See `here `_. """ - log = _ec2_logger() _init = False _client = None @@ -208,43 +191,43 @@ def clean_up(self): vpc = self._resources.Vpc(vpc_id) for instance in vpc.instances.all(): instance.terminate() - self.log.info( + log.info( 'Terminating "%s". This might take a minute.......', instance.id) instance.wait_until_terminated() self._server = None - self.log.info('Instance "%s" terminated.', instance.id) + log.info('Instance "%s" terminated.', instance.id) for s_g in vpc.security_groups.all(): if s_g.group_name == self._remote_config.security_group_name: - self.log.info('Deleting "%s".......', s_g.id) + log.info('Deleting "%s".......', s_g.id) s_g.delete() self._server_security_group = None - self.log.info('Security group "%s" deleted.', s_g.id) + log.info('Security group "%s" deleted.', s_g.id) elif s_g.group_name == 'default': self._default_security_group = None for subnet in vpc.subnets.all(): - self.log.info('Deleting %s.......', subnet.id) + log.info('Deleting %s.......', subnet.id) subnet.delete() self._subnets['public'] = None - self.log.info('Subnet %s deleted.', subnet.id) + log.info('Subnet %s deleted.', subnet.id) for igw in vpc.internet_gateways.all(): - self.log.info('Detaching %s.......', igw.id) + log.info('Detaching %s.......', igw.id) igw.detach_from_vpc(VpcId=vpc.vpc_id) - self.log.info('Gateway %s detached.', igw.id) - self.log.info('Deleting %s.......', igw.id) + log.info('Gateway %s detached.', igw.id) + log.info('Deleting %s.......', igw.id) igw.delete() - self.log.info('Gateway %s deleted.', igw.id) - self.log.info('Deleting %s.......', vpc.id) + log.info('Gateway %s deleted.', igw.id) + log.info('Deleting %s.......', vpc.id) vpc.delete() self._vpc = None - self.log.info('VPC %s deleted.', vpc.id) + log.info('VPC %s deleted.', vpc.id) try: self._client.describe_key_pairs( KeyNames=[self._remote_config.key_name]) key_pair = self._resources.KeyPair( self._remote_config.key_name) - self.log.info( + log.info( 'Deleting "%s".', self._remote_config.key_name) - self.log.info( + log.info( 'Key "%s" deleted.', self._remote_config.key_name) key_pair.delete() except: @@ -259,7 +242,7 @@ def _launch_network(self): """ Launches required network resources. """ - self.log.info("Launching Network.......") + log.info("Launching Network.......") self._create_sssc_vpc() self._create_sssc_subnet(public=True) self._create_sssc_subnet(public=False) @@ -288,10 +271,10 @@ def _delete_root_key(self) -> None: Deletes key from local filesystem if it exists. """ if os.path.exists(self._local_config.key_path): - self.log.info( + log.info( 'Deleting "%s".', self._local_config.key_path) os.remove(self._local_config.key_path) - self.log.info('"%s" deleted.', self._local_config.key_path) + log.info('"%s" deleted.', self._local_config.key_path) def _create_sssc_vpc(self): """ @@ -478,7 +461,7 @@ def _launch_head_node(self, instance_type): 'UserData': launch_commands, } - self.log.info( + log.info( 'Launching StochSS-Compute server instance. This might take a minute.......') try: response = self._client.run_instances(**kwargs) @@ -491,16 +474,16 @@ def _launch_head_node(self, instance_type): self._server.wait_until_exists() self._server.wait_until_running() - self.log.info('Instance "%s" is running.', instance_id) + log.info('Instance "%s" is running.', instance_id) self._poll_launch_progress(['sssc']) - self.log.info('Restricting server access to only your ip.') + log.info('Restricting server access to only your ip.') source_ip = self._get_source_ip(cloud_key) self._restrict_ingress(source_ip) self._init = True - self.log.info('StochSS-Compute ready to go!') + log.info('StochSS-Compute ready to go!') def _poll_launch_progress(self, container_names, mock=False): """ @@ -543,7 +526,7 @@ def _poll_launch_progress(self, container_names, mock=False): "Something went wrong connecting to the server. No exit status provided by the server.") # Wait for yum update, docker install, container download if rc == 1 or rc == 127: - self.log.info('Waiting on Docker daemon.') + log.info('Waiting on Docker daemon.') sshtries += 1 if sshtries >= 5: ssh.close() @@ -552,7 +535,7 @@ def _poll_launch_progress(self, container_names, mock=False): if rc == 0: if 'true\n' in out: sleep(10) - self.log.info('Container "%s" is running.', container) + log.info('Container "%s" is running.', container) break ssh.close() @@ -601,7 +584,7 @@ def _load_cluster(self): pass return False if len(vpc_response['Vpcs']) == 2: - self.log.error('More than one VPC named "%s".', + log.error('More than one VPC named "%s".', self._remote_config.vpc_name) self._set_status('VPC error') raise ResourceException @@ -614,7 +597,7 @@ def _load_cluster(self): if tag['Key'] == 'Name' and tag['Value'] == self._remote_config.server_name: self._server = instance if self._server is None: - self.log.warn('No instances named "%s".', + log.warn('No instances named "%s".', self._remote_config.server_name) self._set_status('server error') errors = True @@ -626,12 +609,12 @@ def _load_cluster(self): if rule['FromPort'] == self._remote_config.api_port \ and rule['ToPort'] == self._remote_config.api_port \ and rule['IpRanges'][0]['CidrIp'] == '0.0.0.0/0': - self.log.warn('Security Group rule error.') + log.warn('Security Group rule error.') self._set_status('security group error') errors = True self._server_security_group = s_g if self._server_security_group is None: - self.log.warn('No security group named "%s".', + log.warn('No security group named "%s".', self._remote_config.security_group_name) self._set_status('security group error') errors = True @@ -642,13 +625,13 @@ def _load_cluster(self): if tag['Key'] == 'Name' and tag['Value'] == f'{self._remote_config.subnet_name}-private': self._subnets['private'] = subnet if None in self._subnets.values(): - self.log.warn('Missing or misconfigured subnet.') + log.warn('Missing or misconfigured subnet.') self._set_status('subnet error') errors = True if errors is True: raise ResourceException else: self._init = True - self.log.info('Cluster loaded.') + log.info('Cluster loaded.') self._set_status(self._server.state['Name']) return True From 0a7840307de41ff27a987ad959af43e31ddf01d1 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 12:59:48 -0400 Subject: [PATCH 41/54] factory method --- gillespy2/remote/core/remote_results.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py index 6982b2c7..702002fb 100644 --- a/gillespy2/remote/core/remote_results.py +++ b/gillespy2/remote/core/remote_results.py @@ -61,6 +61,16 @@ def __init__(self, data = None): self._data = data # pylint:enable=super-init-not-called + @staticmethod + def factory(id, server, n_traj, task_id=None, namespace=None, data=None): + remote_results = RemoteResults(data=data) + remote_results.id = id + remote_results.task_id = task_id + remote_results.server = server + remote_results.n_traj = n_traj + remote_results.namespace = namespace + return remote_results + @property def data(self): """ @@ -149,7 +159,7 @@ def _resolve(self): if status == SimStatus.READY: log.info('Results ready. Fetching.......') - results_request = ResultsRequest(self.id, self.namespace) + results_request = ResultsRequest(self.id, self.namespace, self.n_traj) response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, f"/results", request=results_request) if not response_raw.ok: raise RemoteSimulationError(response_raw.reason) From a5010ee1e1ee4b08534099d819c9cc51798301e3 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:00:12 -0400 Subject: [PATCH 42/54] run_distributed --- gillespy2/remote/core/remote_simulation.py | 159 ++++++++++++++++----- 1 file changed, 121 insertions(+), 38 deletions(-) diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py index f15aef3d..a28aa26b 100644 --- a/gillespy2/remote/core/remote_simulation.py +++ b/gillespy2/remote/core/remote_simulation.py @@ -19,7 +19,7 @@ from gillespy2.remote.client.endpoint import Endpoint from gillespy2.remote.core.messages.simulation_run_cache import SimulationRunCacheRequest, SimulationRunCacheResponse -from gillespy2.remote.core.messages.status import SimStatus +from gillespy2.remote.core.messages.status import SimStatus, StatusRequest from gillespy2.remote.core.errors import RemoteSimulationError from gillespy2.remote.core.remote_results import RemoteResults @@ -37,7 +37,7 @@ class RemoteSimulation: :param server: A server to run the simulation. Optional if host is provided. :type server: gillespy2.remote.Server - :param host: The address of a running instance of StochSS-Compute. Optional if server is provided. + :param host: The address of a running instance of gillespy2.remote. Optional if server is provided. :type host: str :param port: The port to use when connecting to the host. @@ -80,28 +80,25 @@ def is_cached(self, namespace=None, **params): ''' Checks to see if a dummy simulation exists in the cache. + :param namespace: If provided, prepend to results path. + :type namespace: str + :param params: Arguments for simulation. :type params: dict[str, Any] :returns: If the results are cached on the server. :rtype: bool ''' - if "solver" in params: - if hasattr(params['solver'], 'is_instantiated'): - raise RemoteSimulationError( - 'RemoteSimulation does not accept an instantiated solver object. Pass a type.') - params["solver"] = f"{params['solver'].__module__}.{params['solver'].__qualname__}" - if self.solver is not None: - params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" - - sim_request = SimulationRunCacheRequest(model=self.model, namespace=namespace, **params) - results_dummy = RemoteResults() - results_dummy.id = sim_request.results_id - results_dummy.server = self.server - results_dummy.n_traj = params.get('number_of_trajectories', 1) - return results_dummy.is_ready + log.debug('is_cached()') + n_traj_requested = params.get('number_of_trajectories', 1) + n_traj_cached = self.get_n_traj_in_cache(namespace=namespace, + **params) + if n_traj_requested <= n_traj_cached: + return True + else: + return False - def run(self, namespace=None, ignore_cache=False, parallelize=False, submit_chunks=False, **params): + def run(self, namespace=None, ignore_cache=False, parallelize=False, chunk_trajectories=False, **params): # pylint:disable=line-too-long """ Simulate the Model on the target ComputeServer, returning the results or a handle to a running simulation. @@ -123,19 +120,16 @@ def run(self, namespace=None, ignore_cache=False, parallelize=False, submit_chun """ # pylint:enable=line-too-long - if "solver" in params: - if hasattr(params['solver'], 'is_instantiated'): - raise RemoteSimulationError( - 'RemoteSimulation does not accept an instantiated solver object. Pass a type.') - params["solver"] = f"{params['solver'].__module__}.{params['solver'].__qualname__}" - if self.solver is not None: - params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" + params = self._encode_solver_arg(**params) sim_request = SimulationRunCacheRequest(self.model, namespace=namespace, ignore_cache=ignore_cache, parallelize=parallelize, - submit_chunks=submit_chunks, + chunk_trajectories=chunk_trajectories, **params) + log.debug('run()...') + log.debug('sim_request.results_id') + log.debug(sim_request.results_id) return self._run_cache(sim_request) def _run_cache(self, request): @@ -143,6 +137,9 @@ def _run_cache(self, request): :param request: Request to send to the server. Contains Model and related arguments. :type request: SimulationRunRequest ''' + log.debug('_run_cache(request)...') + log.debug('request.kwargs.get("solver", None)') + log.debug(request.kwargs.get('solver', None)) response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub='/run/cache', request=request) if not response_raw.ok: @@ -152,18 +149,104 @@ def _run_cache(self, request): if sim_response.status == SimStatus.ERROR: raise RemoteSimulationError(sim_response.error_message) - # non-conforming object creation ... possible refactor needed to solve, so left in. if sim_response.status == SimStatus.READY: - remote_results = RemoteResults(data=sim_response.results.data) + data = sim_response.results.data else: - remote_results = RemoteResults() - - remote_results.id = request.results_id - remote_results.task_id = request.id - remote_results.server = self.server - remote_results.n_traj = request.kwargs.get('number_of_trajectories', 1) - remote_results.namespace = request.namespace - - return remote_results - -# def run_parallel(self, namespace=None): \ No newline at end of file + data = None + return RemoteResults.factory(request.results_id, + self.server, + request.kwargs.get('number_of_trajectories', 1), + request.id, + request.namespace, + data) + + def run_distributed(self, namespace=None, force_run=False, **params): + log.debug('run_distributed()...') + log.debug('namespace:') + log.debug(namespace) + log.debug('force_run:') + log.debug(force_run) + log.debug('params.get("solver", None)') + log.debug(params.get('solver', None)) + params = self._encode_solver_arg(**params) + log.debug('params.get("solver", None)') + log.debug(params.get('solver', None)) + response_raw = self.server.get(Endpoint.DASK, sub='/number_of_workers') + n_workers = int(response_raw.text) + log.debug('n_workers:') + log.debug(n_workers) + n_traj_requested = params.get('number_of_trajectories', 1) + results_collection = [] + if force_run is True: + n_traj = n_traj_requested + if force_run is False: + n_traj_remote = self.get_n_traj_in_cache(namespace=namespace, + **params) + log.debug('n_traj_remote:') + log.debug(n_traj_remote) + n_traj = n_traj_requested - n_traj_remote + + traj_per_worker = n_traj // n_workers + log.debug('traj_per_worker:') + log.debug(traj_per_worker) + extra_traj = n_traj % n_workers + log.debug('extra_traj:') + log.debug(extra_traj) + if traj_per_worker > 0: + for _ in range(n_workers): + params['number_of_trajectories'] = traj_per_worker + sim_request = SimulationRunCacheRequest(self.model, + namespace=namespace, + force_run=True, + **params) + results_collection.append(self._run_cache(sim_request)) + if extra_traj > 0: + params['number_of_trajectories'] = extra_traj + sim_request = SimulationRunCacheRequest(self.model, + namespace=namespace, + force_run=True, + **params) + results_collection.append(self._run_cache(sim_request)) + + return RemoteResults.factory(sim_request.results_id, + self.server, + n_traj_requested, + None, + sim_request.namespace, + None) + + def get_n_traj_in_cache(self, namespace=None, **params) -> int: + log.debug('get_n_traj_in_cache()...') + log.debug('namespace:') + log.debug(namespace) + params = self._encode_solver_arg(**params) + simulation_request = SimulationRunCacheRequest(self.model, + namespace=namespace, + **params) + log.debug('simulation_request.results_id:') + log.debug(simulation_request.results_id) + n_traj = params.get('number_of_trajectories', 1) + status_request = StatusRequest(results_id=simulation_request.results_id, + n_traj=n_traj, + namespace=namespace) + response_raw = self.server.get(Endpoint.CACHE, sub='/number_of_trajectories', request=status_request) + n_traj_in_cache = int(response_raw.text) + log.debug('n_traj_in_cache:') + log.debug(n_traj_in_cache) + return n_traj_in_cache + + def _encode_solver_arg(self, **params): + log.debug('_encode_solver_arg()...') + value = params.get('solver', '') # Makes for a better fail if they pass something invalid. + log.debug('params.get("solver", None)') + log.debug(params.get('solver', None)) + if type(value) is not str: + if hasattr(params['solver'], 'is_instantiated'): + raise RemoteSimulationError( + 'RemoteSimulation does not accept an instantiated solver object. Pass a type.') + params['solver'] = f"{value.__module__}.{value.__qualname__}" + if self.solver is not None and value == '': + params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" + log.debug('params.get("solver", None)') + log.debug(params.get('solver', None)) + return params From 6b82c3868178aa14a3df8c4adcc68145cd900511 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:00:24 -0400 Subject: [PATCH 43/54] cleanup --- gillespy2/remote/core/messages/results.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/gillespy2/remote/core/messages/results.py b/gillespy2/remote/core/messages/results.py index 0f25dea7..dc657c1e 100644 --- a/gillespy2/remote/core/messages/results.py +++ b/gillespy2/remote/core/messages/results.py @@ -17,9 +17,10 @@ class ResultsRequest(Request): :type namespace: str ''' - def __init__(self, results_id, namespace=None): + def __init__(self, results_id, namespace=None, n_traj=None): self.results_id = results_id self.namespace = namespace + self.n_traj = n_traj def encode(self): ''' @@ -40,7 +41,10 @@ def parse(raw_request): :rtype: ResultsRequest ''' request_dict = json_decode(raw_request) - return ResultsRequest(request_dict['results_id'], request_dict['namespace']) + results_id = request_dict.get('results_id', None) + namespace = request_dict.get('namespace', None) + n_traj = request_dict.get('n_traj', None) + return ResultsRequest(results_id, namespace, n_traj) class ResultsResponse(Response): ''' From 6514095a8af997c0db569aaf31a41ce2f0295d4c Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:00:43 -0400 Subject: [PATCH 44/54] debug and cleanup --- .../core/messages/simulation_run_cache.py | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/gillespy2/remote/core/messages/simulation_run_cache.py b/gillespy2/remote/core/messages/simulation_run_cache.py index ae9b0664..45419ab8 100644 --- a/gillespy2/remote/core/messages/simulation_run_cache.py +++ b/gillespy2/remote/core/messages/simulation_run_cache.py @@ -11,6 +11,9 @@ from gillespy2.remote.core.messages.base import Request, Response from gillespy2.remote.core.messages.status import SimStatus +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + class SimulationRunCacheRequest(Request): ''' A simulation request message object. @@ -24,7 +27,14 @@ class SimulationRunCacheRequest(Request): :param kwargs: kwargs for the model.run() call. :type kwargs: dict[str, Any] ''' - def __init__(self, model, namespace=None, ignore_cache=False, parallelize=False, chunk_trajectories=False, **kwargs): + def __init__(self, + model, + namespace=None, + force_run=False, + ignore_cache=False, + parallelize=False, + chunk_trajectories=False, + **kwargs): self.model = model self.kwargs = kwargs self.id = token_hex(16) @@ -33,6 +43,7 @@ def __init__(self, model, namespace=None, ignore_cache=False, parallelize=False, else: self.results_id = self.hash() self.namespace = namespace + self.force_run = force_run self.ignore_cache = ignore_cache self.parallelize = parallelize self.chunk_trajectories = chunk_trajectories @@ -47,6 +58,7 @@ def encode(self): 'id': self.id, 'results_id': self.results_id, 'namespace': self.namespace, + 'force_run': self.force_run, 'ignore_cache': self.ignore_cache, 'parallelize': self.parallelize, 'chunk_trajectories': self.chunk_trajectories, @@ -73,13 +85,15 @@ def parse(raw_request): id = request_dict.get('id', None) # apply correct token (from raw request) after object construction. results_id = request_dict.get('results_id', None) # apply correct token (from raw request) after object construction. namespace = request_dict.get('namespace', None) + force_run = request_dict.get('force_run', None) ignore_cache = request_dict.get('ignore_cache', None) parallelize = request_dict.get('parallelize', None) chunk_trajectories = request_dict.get('chunk_trajectories', None) - if None in (model, id, results_id, ignore_cache, parallelize, chunk_trajectories): + if None in (model, id, results_id, force_run, ignore_cache, parallelize, chunk_trajectories): raise MessageParseException _ = SimulationRunCacheRequest(model, namespace=namespace, + force_run=force_run, ignore_cache=ignore_cache, parallelize=parallelize, chunk_trajectories=chunk_trajectories, @@ -96,14 +110,19 @@ def hash(self): :returns: md5 hex digest. :rtype: str ''' + log.debug('SimulationRunCacheRequest.hash()...') anon_model_string = self.model.to_anon().to_json(encode_private=False) popped_kwargs = {kw:self.kwargs[kw] for kw in self.kwargs if kw!='number_of_trajectories'} # Explanation of line above: # Take 'self.kwargs' (a dict), and add all entries to a new dictionary, # EXCEPT the 'number_of_trajectories' key/value pair. kwargs_string = json_encode(popped_kwargs) + log.debug('kwargs_string:') + log.debug(kwargs_string) request_string = f'{anon_model_string}{kwargs_string}' _hash = md5(str.encode(request_string)).hexdigest() + log.debug('_hash:') + log.debug(_hash) return _hash From b4b79f588dc82b3472d9e96a341df2a89fba84f6 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:00:54 -0400 Subject: [PATCH 45/54] init --- gillespy2/remote/core/utils/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 gillespy2/remote/core/utils/__init__.py diff --git a/gillespy2/remote/core/utils/__init__.py b/gillespy2/remote/core/utils/__init__.py new file mode 100644 index 00000000..e69de29b From 4b62fa104be61c14c4d413e84123b25ef0b97ff9 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:01:08 -0400 Subject: [PATCH 46/54] silences those logs --- gillespy2/remote/core/utils/dask.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 gillespy2/remote/core/utils/dask.py diff --git a/gillespy2/remote/core/utils/dask.py b/gillespy2/remote/core/utils/dask.py new file mode 100644 index 00000000..e9dc3eba --- /dev/null +++ b/gillespy2/remote/core/utils/dask.py @@ -0,0 +1,25 @@ +import os +message = '''distributed: + version: 2 + logging: + distributed: error + distributed.client: error + distributed.worker: error + distributed.nanny: error + distributed.core: error + ''' +def silence_dask_logs(): + config_path_string = '~/.config/dask/distributed.yaml' + config_path = os.path.expanduser(config_path_string) + if os.path.exists(config_path) is True: + print(f'To silence dask logs, edit the file {config_path} to contain:') + print() + print(message) + else: + config_dir_string = '~/.config/dask/' + config_dir = os.path.expanduser(config_dir_string) + if os.path.exists(config_dir) is False: + os.makedirs(config_dir) + with open(config_path, 'x', encoding='utf-8') as file: + file.write(message) + file.close() From 413fd7ed87cdc5793d28c0a2e802ac58190ff685 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:01:17 -0400 Subject: [PATCH 47/54] typo --- gillespy2/remote/core/utils/debug.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gillespy2/remote/core/utils/debug.py b/gillespy2/remote/core/utils/debug.py index f52d8893..4cae165b 100644 --- a/gillespy2/remote/core/utils/debug.py +++ b/gillespy2/remote/core/utils/debug.py @@ -20,4 +20,4 @@ from logging import getLogger getLogger().setLevel('DEBUG') # Call the line below in a python process to initiate client-side logging -# import gillespy2.remote.core.debug \ No newline at end of file +# import gillespy2.remote.core.utils.debug \ No newline at end of file From 785fcdca105047b6022f5f84809cce69a0fce69f Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:43:06 -0400 Subject: [PATCH 48/54] add handlers --- gillespy2/remote/server/api.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py index 5da8b4eb..d40fee2d 100644 --- a/gillespy2/remote/server/api.py +++ b/gillespy2/remote/server/api.py @@ -20,9 +20,10 @@ import os import asyncio import subprocess -from logging import DEBUG, INFO +from logging import INFO from tornado.web import Application -from gillespy2.remote.server.handlers.is_cached import IsCachedHandler +from gillespy2.remote.server.handlers.number_of_trajectories import NumberOfTrajectoriesHandler +from gillespy2.remote.server.handlers.number_of_workers import NumberOfWorkersHandler from gillespy2.remote.server.handlers.simulation_run_cache import SimulationRunCacheHandler from gillespy2.remote.server.handlers.sourceip import SourceIpHandler from gillespy2.remote.server.handlers.status import StatusHandler @@ -32,22 +33,22 @@ def _make_app(dask_host, dask_scheduler_port, cache): scheduler_address = f'{dask_host}:{dask_scheduler_port}' - args = {'scheduler_address': scheduler_address, - 'cache_dir': cache} + scheduler_arg = {'scheduler_address': scheduler_address} cache_arg = {'cache_dir': cache} + args1 = scheduler_arg | cache_arg return Application([ (r'/api/v3/simulation/gillespy2/run/cache', SimulationRunCacheHandler, - args), + args1), (r'/api/v3/simulation/gillespy2/status', StatusHandler, - args), + args1), (r'/api/v3/simulation/gillespy2/results', ResultsHandler, cache_arg), - (r'/api/v3/cache/gillespy2/(?P.*?)/(?P[1-9]\d*?)/is_cached', - IsCachedHandler, {'cache_dir': cache}), (r'/api/v3/cloud/sourceip', SourceIpHandler), + (r'/api/v3/dask/number_of_workers', NumberOfWorkersHandler, scheduler_arg), + (r'/api/v3/cache/number_of_trajectories', NumberOfTrajectoriesHandler, cache_arg), ]) async def start_api( @@ -84,9 +85,7 @@ async def start_api( """ set_global_log_level(logging_level) - # TODO clean up lock files here - # cache_path = os.path.abspath(cache_path) app = _make_app(dask_host, dask_scheduler_port, cache_path) app.listen(port) msg=''' From 37147829447813194eb30ead0312d03a1495abc4 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:43:23 -0400 Subject: [PATCH 49/54] cleanup --- gillespy2/remote/server/cache.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/gillespy2/remote/server/cache.py b/gillespy2/remote/server/cache.py index 55c3b94d..c9cb8177 100644 --- a/gillespy2/remote/server/cache.py +++ b/gillespy2/remote/server/cache.py @@ -19,7 +19,6 @@ import os from json.decoder import JSONDecodeError -from datetime import datetime import random from filelock import SoftFileLock from gillespy2 import Results @@ -36,6 +35,9 @@ class Cache: :param results_id: Simulation hash. :type results_id: str + + :param namespace: Optional namespace. + :type namespace: str ''' def __init__(self, cache_dir, results_id, namespace=None): if namespace is not None: @@ -83,16 +85,18 @@ def is_ready(self, n_traj_wanted=0) -> bool: ''' Check if the results are ready to be retrieved from the cache. - :param n_traj_wanted: The number of requested trajectories. + :param n_traj_wanted: The number of requested trajectories. If set to 0, check if any. :type: int :returns: n_traj_wanted <= len() :rtype: bool ''' - results = self.get() - if results is None or n_traj_wanted > len(results): - return False - return True + len_results = self.n_traj_in_cache() + if n_traj_wanted == 0 and len_results > 0: + return True + if n_traj_wanted <= len_results: + return True + return False def n_traj_needed(self, n_traj_wanted) -> int: ''' From 4c1d786075381095425ba32b8617843132440869 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:43:31 -0400 Subject: [PATCH 50/54] new handler --- .../server/handlers/number_of_trajectories.py | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 gillespy2/remote/server/handlers/number_of_trajectories.py diff --git a/gillespy2/remote/server/handlers/number_of_trajectories.py b/gillespy2/remote/server/handlers/number_of_trajectories.py new file mode 100644 index 00000000..be3e3b41 --- /dev/null +++ b/gillespy2/remote/server/handlers/number_of_trajectories.py @@ -0,0 +1,62 @@ +''' +gillespy2.remote.server.number_of_trajectories +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tornado.web import RequestHandler + +from gillespy2.remote.core.utils.log_config import init_logging +from gillespy2.remote.server.cache import Cache +log = init_logging(__name__) + +class NumberOfTrajectoriesHandler(RequestHandler): + ''' + Responds with the number of trajectories in the cache. + ''' + def __init__(self, application, request, **kwargs): + self.cache_dir = None + super().__init__(application, request, **kwargs) + + def data_received(self, chunk: bytes): + raise NotImplementedError() + + def initialize(self, cache_dir): + ''' + Set the cache directory. + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + self.cache_dir = cache_dir + + def get(self): + ''' + Process GET request. + + :returns: Number of trajectories in the cache. + :rtype: str + ''' + msg = f'Request from {self.request.remote_ip}' + log.info(msg) + results_id = self.get_query_argument('results_id') + namespace = self.get_query_argument('namespace', None) + msg = f'results_id: <{results_id}> | namespace: <{namespace}>' + log.info(msg) + cache = Cache(self.cache_dir, results_id, namespace=namespace) + n_traj_in_cache = cache.n_traj_in_cache() + self.write(str(n_traj_in_cache)) + self.finish() From ff192b11e16b5d5602b826f05ad0d11858779696 Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:43:46 -0400 Subject: [PATCH 51/54] new handler --- .../server/handlers/number_of_workers.py | 57 +++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 gillespy2/remote/server/handlers/number_of_workers.py diff --git a/gillespy2/remote/server/handlers/number_of_workers.py b/gillespy2/remote/server/handlers/number_of_workers.py new file mode 100644 index 00000000..11d6b8f6 --- /dev/null +++ b/gillespy2/remote/server/handlers/number_of_workers.py @@ -0,0 +1,57 @@ +''' +gillespy2.remote.server.sourceip +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tornado.web import RequestHandler +from distributed import Client + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class NumberOfWorkersHandler(RequestHandler): + ''' + Responds with the number of workers attached to the scheduler. + ''' + scheduler_address = None + def initialize(self, scheduler_address): + ''' + Sets the address to the Dask scheduler and the cache directory. + + :param scheduler_address: Scheduler address. + :type scheduler_address: str + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + + self.scheduler_address = scheduler_address + + def get(self): + ''' + Process POST request. + + :returns: request.remote_ip + :rtype: str + ''' + msg = f' <{self.request.remote_ip}>' + log.info(msg) + client = Client(self.scheduler_address) + n_workers = len(client.scheduler_info().get('workers', [])) + self.write(str(n_workers)) + self.finish() + client.close() From b0e57231ff17cd59c23e177c175074f9c3776f8c Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:44:09 -0400 Subject: [PATCH 52/54] debug --- gillespy2/remote/server/handlers/results.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/gillespy2/remote/server/handlers/results.py b/gillespy2/remote/server/handlers/results.py index 767d69cf..e70dab9a 100644 --- a/gillespy2/remote/server/handlers/results.py +++ b/gillespy2/remote/server/handlers/results.py @@ -42,8 +42,6 @@ def initialize(self, cache_dir): :param cache_dir: Path to the cache. :type cache_dir: str ''' - # while cache_dir.endswith('/'): - # cache_dir = cache_dir[:-1] self.cache_dir = cache_dir async def get(self): @@ -58,6 +56,7 @@ async def get(self): ''' results_id = self.get_query_argument('results_id', None) n_traj = self.get_query_argument('n_traj', 0) + n_traj = int(n_traj) namespace = self.get_query_argument('namespace', None) if results_id is None: self.set_status(404, reason=f'Malformed request: {self.request.uri}') @@ -67,13 +66,16 @@ async def get(self): log.info(msg) cache = Cache(self.cache_dir, results_id, namespace=namespace) if cache.is_ready(n_traj_wanted=n_traj): + log.debug('cache.is_ready()...') + log.debug('n_traj') + log.debug(n_traj) if n_traj > 0: results = cache.get_sample(n_traj) + results = results.to_json() else: results = cache.read() results_response = ResultsResponse(results) self.write(results_response.encode()) else: - # This should not happen! self.set_status(404, f'Results "{results_id}" not found.') self.finish() From d4a4f5b7e6d52e4e90cfe5bdc8c22997e19eb18e Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:46:14 -0400 Subject: [PATCH 53/54] cleanup and debug --- .../server/handlers/simulation_run_cache.py | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/gillespy2/remote/server/handlers/simulation_run_cache.py b/gillespy2/remote/server/handlers/simulation_run_cache.py index df2b2e32..a7ce62e4 100644 --- a/gillespy2/remote/server/handlers/simulation_run_cache.py +++ b/gillespy2/remote/server/handlers/simulation_run_cache.py @@ -58,8 +58,11 @@ async def post(self): Process simulation run request. ''' sim_request = SimulationRunCacheRequest.parse(self.request.body) - log.debug(sim_request.encode()) - log.debug('%(namespace)s', locals()) + log.debug('sim_request...') + log.debug('.results_id:') + log.debug(sim_request.results_id) + log.debug('.id:') + log.debug(sim_request.id) results_id = sim_request.results_id self.cache = Cache(self.cache_dir, results_id, namespace=sim_request.namespace) @@ -67,14 +70,18 @@ async def post(self): if not self.cache.exists(): self.cache.create() empty = self.cache.is_empty() - if not empty: - # Check the number of trajectories in the request, default 1 - n_traj = sim_request.kwargs.get('number_of_trajectories', 1) + # Check the number of trajectories in the request, default 1 + n_traj = sim_request.kwargs.get('number_of_trajectories', 1) + if sim_request.force_run is True: + msg = f'{msg_0} | Force run simulation. Running {n_traj} new trajectories.' + log.info(msg) + self._run_cache(sim_request) + elif not empty: # Compare that to the number of cached trajectories trajectories_needed = self.cache.n_traj_needed(n_traj) if trajectories_needed > 0: sim_request.kwargs['number_of_trajectories'] = trajectories_needed - msg = f'{msg_0} | Partial self.. Running {trajectories_needed} new trajectories.' + msg = f'{msg_0} | Partial cache. Running {trajectories_needed} new trajectories.' log.info(msg) self._run_cache(sim_request) else: @@ -85,7 +92,7 @@ async def post(self): sim_response = SimulationRunCacheResponse(SimStatus.READY, results_id = results_id, results = results_json) self.write(sim_response.encode()) self.finish() - if empty: + elif empty: msg = f'{msg_0} | Results not cached. Running simulation.' log.info(msg) self._run_cache(sim_request) @@ -114,7 +121,7 @@ def _run_cache(self, sim_request) -> None: else: future = self._submit(sim_request, client) self._return_running(results_id, future.key) - IOLoop.current().run_in_executor(None, self._cache, results_id, future, client) + IOLoop.current().run_in_executor(None, self._cache, future, client) def _submit_parallel(self, sim_request, client: Client): ''' @@ -142,7 +149,7 @@ def _submit_parallel(self, sim_request, client: Client): kwargs['seed'] += kwargs['seed'] future = client.submit(model.run, **kwargs) futures.append(future) - IOLoop.current().run_in_executor(None, self._cache_parallel, results_id, futures, client) + IOLoop.current().run_in_executor(None, self._cache_parallel, futures, client) self._return_running(results_id, results_id) def _submit_chunks(self, sim_request, client: Client): @@ -169,8 +176,6 @@ def _submit_chunks(self, sim_request, client: Client): log.debug('_submit_chunks():') log.debug(traj_per_worker) log.debug(extra_traj) - # return - # traj_per_worker_list = [] futures = [] del kwargs['number_of_trajectories'] kwargs['seed'] = int(sim_request.id, 16) @@ -185,12 +190,12 @@ def _submit_chunks(self, sim_request, client: Client): future = client.submit(model.run, **kwargs) futures.append(future) - IOLoop.current().run_in_executor(None, self._cache_parallel, results_id, futures, client) + IOLoop.current().run_in_executor(None, self._cache_parallel, futures, client) self._return_running(results_id, results_id) - def _cache_parallel(self, results_id, futures, client: Client) -> None: + def _cache_parallel(self, futures, client: Client) -> None: ''' :param results_id: Key to results. :type results_id: str @@ -202,20 +207,16 @@ def _cache_parallel(self, results_id, futures, client: Client) -> None: :type client: distributed.Client ''' + log.debug('_cache_parallel()...') results = client.gather(futures, asynchronous=False) results = sum(results) - log.debug('_cache_parallel():') + log.debug('results:') log.debug(results) client.close() - # return - # cache = Cache(self.cache_dir, results_id) self.cache.save(results) - def _cache(self, results_id, future, client) -> None: + def _cache(self, future, client) -> None: ''' - :param results_id: Key to results. - :type results_id: str - :param future: Future that completes to gillespy2.Results. :type future: distributed.Future @@ -225,7 +226,6 @@ def _cache(self, results_id, future, client) -> None: ''' results = future.result() client.close() - # cache = Cache(self.cache_dir, results_id) self.cache.save(results) def _submit(self, sim_request, client: Client): @@ -239,12 +239,15 @@ def _submit(self, sim_request, client: Client): :returns: Future that completes to gillespy2.Results. :rtype: distributed.Future ''' + log.debug('_submit()...') model = sim_request.model kwargs = sim_request.kwargs if "solver" in kwargs: from pydoc import locate kwargs["solver"] = locate(kwargs["solver"]) + log.debug('kwargs.get("solver", None)') + log.debug(kwargs.get('solver', None)) return client.submit(model.run, **kwargs, key=sim_request.id) From 6cd00c1ee12e765b454b2264950629556e210cce Mon Sep 17 00:00:00 2001 From: Matthew Dippel Date: Mon, 24 Jul 2023 13:46:32 -0400 Subject: [PATCH 54/54] cleanup --- gillespy2/remote/server/handlers/sourceip.py | 11 +++++++++-- gillespy2/remote/server/handlers/status.py | 19 +++++++------------ 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/gillespy2/remote/server/handlers/sourceip.py b/gillespy2/remote/server/handlers/sourceip.py index d968ee1e..d63c6a72 100644 --- a/gillespy2/remote/server/handlers/sourceip.py +++ b/gillespy2/remote/server/handlers/sourceip.py @@ -21,6 +21,9 @@ from tornado.web import RequestHandler from gillespy2.remote.core.messages.source_ip import SourceIpRequest, SourceIpResponse +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + class SourceIpHandler(RequestHandler): ''' Responds with the IP address associated with the request. @@ -35,12 +38,16 @@ def post(self): :rtype: str ''' source_ip = self.request.remote_ip - print(f'[SourceIp Request] | Source: <{source_ip}>') + msg = f'Request from {source_ip}' + log.info(msg) source_ip_request = SourceIpRequest.parse(self.request.body) - # could possibly also check just to see if request is valid? if source_ip_request.cloud_key == os.environ.get('CLOUD_LOCK'): + msg = f'{source_ip} Access granted.' + log.info(msg) source_ip_response = SourceIpResponse(source_ip=source_ip) self.write(source_ip_response.encode()) else: + msg = f'{source_ip} Access denied.' + log.info(msg) self.set_status(403, 'Access denied.') self.finish() diff --git a/gillespy2/remote/server/handlers/status.py b/gillespy2/remote/server/handlers/status.py index 8a0ea9c7..4c9f5039 100644 --- a/gillespy2/remote/server/handlers/status.py +++ b/gillespy2/remote/server/handlers/status.py @@ -16,11 +16,9 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . -import os from distributed import Client from tornado.web import RequestHandler -from gillespy2.remote.core.errors import RemoteSimulationError -from gillespy2.remote.core.messages.status import SimStatus, StatusRequest, StatusResponse +from gillespy2.remote.core.messages.status import SimStatus, StatusResponse from gillespy2.remote.server.cache import Cache from gillespy2.remote.core.utils.log_config import init_logging @@ -55,12 +53,8 @@ async def get(self): ''' Process Status GET request. ''' - - # status_request = StatusRequest.parse(self.request) - log.debug(self.request.query_arguments) - results_id = self.get_query_argument('results_id') - n_traj = self.get_query_argument('n_traj', None) + n_traj = self.get_query_argument('n_traj', 0) if n_traj is not None: n_traj = int(n_traj) task_id = self.get_query_argument('task_id', None) @@ -72,6 +66,7 @@ async def get(self): log.info(msg_0) msg_1 = f'<{results_id}> | <{task_id}> | Status:' + running_msg = f'{msg_1} {SimStatus.RUNNING.name}' dne_msg = f'{msg_1} {SimStatus.DOES_NOT_EXIST.name}' ready_msg = f'{msg_1} {SimStatus.READY.name}' @@ -94,8 +89,8 @@ async def get(self): else: - log.info(dne_msg) - self._respond_dne() + log.info(running_msg) + self._respond_running('Simulation still running.') else: @@ -119,8 +114,8 @@ async def get(self): else: - log.info(dne_msg) - self._respond_dne() + log.info(running_msg) + self._respond_running('Simulation still running.') else: log.debug('cache.exists(): False')