diff --git a/.gitignore b/.gitignore index a42edd12..ab886c4b 100644 --- a/.gitignore +++ b/.gitignore @@ -70,4 +70,4 @@ __pycache__ htmlcov /build .idea - +env/ diff --git a/gillespy2/remote/__init__.py b/gillespy2/remote/__init__.py new file mode 100644 index 00000000..db7a59bd --- /dev/null +++ b/gillespy2/remote/__init__.py @@ -0,0 +1,23 @@ +''' +gillespy2.remote +''' +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + + +from gillespy2.remote.client import * +from gillespy2.remote.core import * +from gillespy2.remote.server import * +from gillespy2.remote.cloud import * diff --git a/gillespy2/remote/__version__.py b/gillespy2/remote/__version__.py new file mode 100644 index 00000000..a351a250 --- /dev/null +++ b/gillespy2/remote/__version__.py @@ -0,0 +1,38 @@ +# ''' +# gillespy2.remote.__version__ +# ''' +# # StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# # Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# # This program is free software: you can redistribute it and/or modify +# # it under the terms of the GNU General Public License as published by +# # the Free Software Foundation, either version 3 of the License, or +# # (at your option) any later version. + +# # This program is distributed in the hope that it will be useful, +# # but WITHOUT ANY WARRANTY; without even the implied warranty of +# # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# # GNU General Public License for more details. + +# # You should have received a copy of the GNU General Public License +# # along with this program. If not, see . + + +# # ============================================================================= +# # @file __version__.py +# # @brief stochss-compute version info +# # @license Please see the file named LICENSE.md in parent directory +# # @website https://github.com/StochSS/stochss-compute +# # ============================================================================= + + +# __version__ = '1.0.1' + +# __title__ = "stochss-compute" +# __description__ = "A compute delegation package for the StochSS family of stochastic simulators" +# __url__ = "https://github.com/StochSS/stochss-compute" +# __download_url__ = "https://pypi.org/project/stochss-compute/#files" +# __author__ = "See CONTRIBUTORS.md" +# __email__ = "briandrawert@gmail.com" +# __license__ = "GPLv3" +# __copyright__ = "Copyright (C) 2017-2023" diff --git a/gillespy2/remote/client/__init__.py b/gillespy2/remote/client/__init__.py new file mode 100644 index 00000000..773e0615 --- /dev/null +++ b/gillespy2/remote/client/__init__.py @@ -0,0 +1,19 @@ +''' +gillespy2.remote.client +''' +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .compute_server import ComputeServer diff --git a/gillespy2/remote/client/compute_server.py b/gillespy2/remote/client/compute_server.py new file mode 100644 index 00000000..d3140542 --- /dev/null +++ b/gillespy2/remote/client/compute_server.py @@ -0,0 +1,46 @@ +''' +ComputeServer(Server) +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from gillespy2.remote.client.server import Server + +class ComputeServer(Server): + ''' + Simple object representing a remote instance of StochSS-Compute. + + :param host: Address of the remote server. + :type host: str + + :param port: Port on which to connect. Defaults to 29681. + :type port: int + ''' + # pylint: disable=super-init-not-called + def __init__(self, host: str, port: int = 29681): + host = host.replace('http://','') + host = host.split(':')[0] + self._address = f"http://{host}:{port}" + # pylint: enable=super-init-not-called + + @property + def address(self) -> str: + """ + The server's IP address and port. + + :returns: "http://{host}:{port}" + """ + return self._address diff --git a/gillespy2/remote/client/endpoint.py b/gillespy2/remote/client/endpoint.py new file mode 100644 index 00000000..be8cdf7c --- /dev/null +++ b/gillespy2/remote/client/endpoint.py @@ -0,0 +1,30 @@ +''' +Endpoint(Enum) +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from enum import Enum + +class Endpoint(Enum): + ''' + API Endpoints. + ''' + SIMULATION_GILLESPY2 = 1 + CLOUD = 2 + CACHE = 3 + DASK = 4 + \ No newline at end of file diff --git a/gillespy2/remote/client/server.py b/gillespy2/remote/client/server.py new file mode 100644 index 00000000..99021ba7 --- /dev/null +++ b/gillespy2/remote/client/server.py @@ -0,0 +1,135 @@ +''' +Server(ABC) +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from time import sleep +from abc import ABC, abstractmethod +import requests +from gillespy2.remote.client.endpoint import Endpoint +from gillespy2.remote.core.messages.base import Request + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class Server(ABC): + ''' + Abstract Server class with hard coded endpoints. + + :raises TypeError: Server cannot be instantiated directly. Must be ComputeServer or Cluster. + ''' + + _endpoints = { + Endpoint.SIMULATION_GILLESPY2: "/api/v3/simulation/gillespy2", + Endpoint.CLOUD: "/api/v3/cloud", + Endpoint.CACHE: "/api/v3/cache", + Endpoint.DASK: '/api/v3/dask', + } + + def __init__(self) -> None: + raise TypeError('Server cannot be instantiated directly. Must be ComputeServer or Cluster.') + + @property + @abstractmethod + def address(self): + ''' + NotImplemented + ''' + return NotImplemented + + def get(self, endpoint: Endpoint, sub: str, request: Request = None): + ''' + Send a GET request to endpoint. + + :param endpoint: The API endpoint. + :type endpoint: Endpoint + + :param sub: Final part of url string. + :type sub: str + + :param request: An object that inherits from Request. + :type request: Request + + :returns: The HTTP response. + :rtype: requests.Response + ''' + if request is not None: + log.debug(request.encode()) + url = f"{self.address}{self._endpoints[endpoint]}{sub}" + log.debug(url) + n_try = 1 + sec = 3 + while n_try <= 3: + try: + if request is not None: + return requests.get(url, timeout=30, params=request.encode()) + return requests.get( url, timeout=30) + + except ConnectionError: + print(f"Connection refused by server. Retrying in {sec} seconds....") + sleep(sec) + n_try += 1 + sec *= n_try + except Exception as err: + print(f"Unknown error: {err}. Retrying in {sec} seconds....") + sleep(sec) + n_try += 1 + sec *= n_try + + def post(self, endpoint: Endpoint, sub: str, request: Request = None): + ''' + Send a POST request to endpoint. + + :param endpoint: The API endpoint. + :type endpoint: Endpoint + + :param sub: Final part of url string. + :type sub: str + + :param request: An object that inherits from Request. + :type request: Request + + :returns: The HTTP response. + :rtype: requests.Response + ''' + log.debug(request.encode()) + + if self.address is NotImplemented: + raise NotImplementedError + + url = f"{self.address}{self._endpoints[endpoint]}{sub}" + log.debug(url) + n_try = 1 + sec = 3 + while n_try <= 3: + try: + if request is None: + print(f"[POST] {url}") + return requests.post(url, timeout=30) + print(f"[{type(request).__name__}] {url}") + return requests.post(url, json=request.encode(), timeout=30) + + except ConnectionError: + print(f"Connection refused by server. Retrying in {sec} seconds....") + sleep(sec) + n_try += 1 + sec *= n_try + except Exception as err: + print(f"Unknown error: {err}. Retrying in {sec} seconds....") + sleep(sec) + n_try += 1 + sec *= n_try diff --git a/gillespy2/remote/cloud/__init__.py b/gillespy2/remote/cloud/__init__.py new file mode 100644 index 00000000..dfea54d0 --- /dev/null +++ b/gillespy2/remote/cloud/__init__.py @@ -0,0 +1,21 @@ +''' +gillespy2.remote.cloud +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .ec2 import EC2Cluster +from .ec2_config import EC2LocalConfig, EC2RemoteConfig diff --git a/gillespy2/remote/cloud/ec2.py b/gillespy2/remote/cloud/ec2.py new file mode 100644 index 00000000..247c1e33 --- /dev/null +++ b/gillespy2/remote/cloud/ec2.py @@ -0,0 +1,637 @@ +''' +gillespy2.remote.cloud.ec2 +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import logging +from time import sleep +from secrets import token_hex +from gillespy2.remote.client.server import Server +from gillespy2.remote.cloud.ec2_config import EC2LocalConfig, EC2RemoteConfig +from gillespy2.remote.core.messages.source_ip import SourceIpRequest, SourceIpResponse +from gillespy2.remote.cloud.exceptions import ResourceException, EC2Exception +from gillespy2.remote.client.endpoint import Endpoint +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) +try: + import boto3 + from botocore.config import Config + from botocore.session import get_session + from botocore.exceptions import ClientError + from paramiko import SSHClient, AutoAddPolicy +except ImportError as err: + name = __name__ + log.warn('boto3 and paramiko are required for %(name)s', locals()) + +class EC2Cluster(Server): + """ + Attempts to load a StochSS-Compute cluster. Otherwise just initializes a new cluster. + + :param local_config: Optional. Allows configuration of local cluster resources. + :type local_config: EC2LocalConfig + + :param remote_config: Optional. Allows configuration of remote cluster resource identifiers. + :type remote_config: EC2RemoteConfig + + :raises EC2Exception: possible boto3 ClientError from AWS calls. See `here `_. + """ + + _init = False + _client = None + _resources = None + _restricted: bool = False + _subnets = { + 'public': None, + 'private': None + } + _default_security_group = None + _server_security_group = None + _vpc = None + _server = None + _ami = None + + _local_config = EC2LocalConfig() + _remote_config = EC2RemoteConfig() + + def __init__(self, local_config=None, remote_config=None) -> None: + if local_config is not None: + self._local_config = local_config + if remote_config is not None: + self._remote_config = remote_config + + if self._remote_config.region is not None: + config = Config(region_name=self._remote_config.region) + region = self._remote_config.region + # Overrides any underlying configurationz + self._client = boto3.client('ec2', config=config) + self._resources = boto3.resource('ec2', config=config) + else: + region = get_session().get_config_variable('region') + self._client = boto3.client('ec2') + self._resources = boto3.resource('ec2') + + if self._remote_config.ami is not None: + self._ami = self._remote_config.ami + else: + try: + self._ami = self._remote_config._AMIS[region] + except KeyError as err2: + self._set_status('region error') + raise EC2Exception(f'Unsupported region. Currently Supported: \ + {list(self._remote_config._AMIS.keys())}. \ + Try providing an AMI identifier.') from err2 + + try: + self._load_cluster() + except ClientError as c_e: + self._set_status(c_e.response['Error']['Code']) + raise EC2Exception(c_e.response['Error']['Message']) from c_e + except ResourceException: + self.clean_up() + + @property + def address(self) -> str: + """ + The server's IP address and port. + + :returns: "http://{ip}:{port}" + :rtype: str + + :raises EC2Exception: Do not call before launching a cluster. + """ + if self._server is None: + raise EC2Exception('No server found. First launch a cluster.') + if self._server.public_ip_address is None: + self._server.reload() + if self._server.public_ip_address is None: + raise EC2Exception('No public address found.') + + return f'http://{self._server.public_ip_address}:{self._remote_config.api_port}' + + @property + def status(self) -> str: + ''' + Return the EC2 instance status. + + :returns: A status set locally, or, if connected, a status fetched from the instance. + :rtype: str + ''' + if self._server is None: + return self._status + else: + return self._server.state['Name'] + + def _set_status(self, status): + self._status = status + if self._local_config.status_file is not None: + with open(self._local_config.status_file, 'w', encoding='utf-8') as file: + file.write(status) + + def launch_single_node_instance(self, instance_type): + """ + Launches a single node StochSS-Compute instance. Make sure to check instance_type pricing before launching. + + :param instance_type: Example: 't3.nano' See full list `here `_. + :type instance_type: str + + :raises EC2Exception: possible boto3 ClientError from AWS calls. See `here `_. + """ + if self._init is True: + raise EC2Exception('You cannot launch more than one \ + StochSS-Compute cluster instance \ + per EC2Cluster object.') + + self._set_status('launching') + try: + self._launch_network() + self._create_root_key() + self._launch_head_node(instance_type=instance_type) + except ClientError as c_e: + self._set_status(c_e.response['Error']['Code']) + raise EC2Exception(c_e.response['Error']['Message']) from c_e + self._set_status(self._server.state['Name']) + + def clean_up(self): + """ + Terminates and removes all cluster resources. + + :raises EC2Exception: possible boto3 ClientError from AWS calls. See `here `_. + """ + self._set_status('terminating') + self._init = False + + vpc_search_filter = [ + { + 'Name': 'tag:Name', + 'Values': [ + self._remote_config.vpc_name + ] + } + ] + try: + vpc_response = self._client.describe_vpcs( + Filters=vpc_search_filter) + for vpc_dict in vpc_response['Vpcs']: + vpc_id = vpc_dict['VpcId'] + vpc = self._resources.Vpc(vpc_id) + for instance in vpc.instances.all(): + instance.terminate() + log.info( + 'Terminating "%s". This might take a minute.......', instance.id) + instance.wait_until_terminated() + self._server = None + log.info('Instance "%s" terminated.', instance.id) + for s_g in vpc.security_groups.all(): + if s_g.group_name == self._remote_config.security_group_name: + log.info('Deleting "%s".......', s_g.id) + s_g.delete() + self._server_security_group = None + log.info('Security group "%s" deleted.', s_g.id) + elif s_g.group_name == 'default': + self._default_security_group = None + for subnet in vpc.subnets.all(): + log.info('Deleting %s.......', subnet.id) + subnet.delete() + self._subnets['public'] = None + log.info('Subnet %s deleted.', subnet.id) + for igw in vpc.internet_gateways.all(): + log.info('Detaching %s.......', igw.id) + igw.detach_from_vpc(VpcId=vpc.vpc_id) + log.info('Gateway %s detached.', igw.id) + log.info('Deleting %s.......', igw.id) + igw.delete() + log.info('Gateway %s deleted.', igw.id) + log.info('Deleting %s.......', vpc.id) + vpc.delete() + self._vpc = None + log.info('VPC %s deleted.', vpc.id) + try: + self._client.describe_key_pairs( + KeyNames=[self._remote_config.key_name]) + key_pair = self._resources.KeyPair( + self._remote_config.key_name) + log.info( + 'Deleting "%s".', self._remote_config.key_name) + log.info( + 'Key "%s" deleted.', self._remote_config.key_name) + key_pair.delete() + except: + pass + except ClientError as c_e: + self._set_status(c_e.response['Error']['Code']) + raise EC2Exception(c_e.response['Error']['Message']) from c_e + self._delete_root_key() + self._set_status('terminated') + + def _launch_network(self): + """ + Launches required network resources. + """ + log.info("Launching Network.......") + self._create_sssc_vpc() + self._create_sssc_subnet(public=True) + self._create_sssc_subnet(public=False) + self._create_sssc_security_group() + self._vpc.reload() + + def _create_root_key(self): + """ + Creates a key pair for SSH login and instance launch. + """ + + response = self._client.create_key_pair( + KeyName=self._remote_config.key_name, + KeyType=self._local_config.key_type, + KeyFormat=self._local_config.key_format) + + waiter = self._client.get_waiter('key_pair_exists') + waiter.wait(KeyNames=[self._remote_config.key_name]) + os.makedirs(self._local_config.key_dir, exist_ok=True) + with open(self._local_config.key_path, 'x', encoding='utf-8') as key: + key.write(response['KeyMaterial']) + os.chmod(self._local_config.key_path, 0o400) + + def _delete_root_key(self) -> None: + """ + Deletes key from local filesystem if it exists. + """ + if os.path.exists(self._local_config.key_path): + log.info( + 'Deleting "%s".', self._local_config.key_path) + os.remove(self._local_config.key_path) + log.info('"%s" deleted.', self._local_config.key_path) + + def _create_sssc_vpc(self): + """ + Creates a vpc. + """ + vpc_cidr_block = '172.31.0.0/16' + vpc_tag = [ + { + 'ResourceType': 'vpc', + 'Tags': [ + { + 'Key': 'Name', + 'Value': self._remote_config.vpc_name + } + ] + } + ] + + vpc_response = self._client.create_vpc( + CidrBlock=vpc_cidr_block, TagSpecifications=vpc_tag) + vpc_id = vpc_response['Vpc']['VpcId'] + vpc_waiter_exist = self._client.get_waiter('vpc_exists') + vpc_waiter_exist.wait(VpcIds=[vpc_id]) + vpc_waiter_avail = self._client.get_waiter('vpc_available') + vpc_waiter_avail.wait(VpcIds=[vpc_id]) + self._vpc = self._resources.Vpc(vpc_id) + self._default_security_group = list( + sg for sg in self._vpc.security_groups.all())[0] + + self._client.modify_vpc_attribute( + VpcId=vpc_id, EnableDnsSupport={'Value': True}) + self._client.modify_vpc_attribute( + VpcId=vpc_id, EnableDnsHostnames={'Value': True}) + + igw_response = self._client.create_internet_gateway() + igw_id = igw_response['InternetGateway']['InternetGatewayId'] + igw_waiter = self._client.get_waiter('internet_gateway_exists') + igw_waiter.wait(InternetGatewayIds=[igw_id]) + + self._vpc.attach_internet_gateway(InternetGatewayId=igw_id) + for rtb in self._vpc.route_tables.all(): + if rtb.associations_attribute[0]['Main'] is True: + rtb_id = rtb.route_table_id + self._client.create_route( + RouteTableId=rtb_id, GatewayId=igw_id, DestinationCidrBlock='0.0.0.0/0') + + self._vpc.reload() + + def _create_sssc_subnet(self, public: bool): + """ + Creates a public or private subnet. + """ + if public is True: + label = 'public' + subnet_cidr_block = '172.31.0.0/20' + else: + label = 'private' + subnet_cidr_block = '172.31.16.0/20' + + subnet_tag = [ + { + 'ResourceType': 'subnet', + 'Tags': [ + { + 'Key': 'Name', + 'Value': f'{self._remote_config.subnet_name}-{label}' + } + ] + } + ] + self._subnets[label] = self._vpc.create_subnet( + CidrBlock=subnet_cidr_block, TagSpecifications=subnet_tag) + waiter = self._client.get_waiter('subnet_available') + waiter.wait(SubnetIds=[self._subnets[label].id]) + self._client.modify_subnet_attribute( + SubnetId=self._subnets[label].id, MapPublicIpOnLaunch={'Value': True}) + self._subnets[label].reload() + + def _create_sssc_security_group(self): + """ + Creates a security group for SSH and StochSS-Compute API access. + """ + description = 'Default Security Group for StochSS-Compute.' + self._server_security_group = self._vpc.create_security_group( + Description=description, GroupName=self._remote_config.security_group_name) + sshargs = { + 'CidrIp': '0.0.0.0/0', + 'FromPort': 22, + 'ToPort': 22, + 'IpProtocol': 'tcp', + } + self._server_security_group.authorize_ingress(**sshargs) + sgargs = { + 'CidrIp': '0.0.0.0/0', + 'FromPort': self._remote_config.api_port, + 'ToPort': self._remote_config.api_port, + 'IpProtocol': 'tcp', + 'TagSpecifications': [ + { + 'ResourceType': 'security-group-rule', + 'Tags': [ + { + 'Key': 'Name', + 'Value': 'api-server' + }, + ] + }, + ] + } + self._server_security_group.authorize_ingress(**sgargs) + self._server_security_group.reload() + + def _restrict_ingress(self, ip_address: str = ''): + """ + Modifies the security group API ingress rule to + only allow access on the specified port from the given ip address. + """ + rule_filter = [ + { + 'Name': 'group-id', + 'Values': [ + self._server_security_group.id, + ] + }, + { + 'Name': 'tag:Name', + 'Values': [ + 'api-server', + ] + }, + ] + sgr_response = self._client.describe_security_group_rules( + Filters=rule_filter) + sgr_id = sgr_response['SecurityGroupRules'][0]['SecurityGroupRuleId'] + new_sg_rules = [ + { + 'SecurityGroupRuleId': sgr_id, + 'SecurityGroupRule': { + 'IpProtocol': 'tcp', + 'FromPort': self._remote_config.api_port, + 'ToPort': self._remote_config.api_port, + 'CidrIpv4': f'{ip_address}/32', + 'Description': 'Restricts cluster access.' + } + }, + ] + self._client.modify_security_group_rules( + GroupId=self._server_security_group.id, SecurityGroupRules=new_sg_rules) + self._server_security_group.reload() + + def _launch_head_node(self, instance_type): + """ + Launches a StochSS-Compute server instance. + """ + cloud_key = token_hex(32) + + launch_commands = f'''#!/bin/bash +sudo yum update -y +sudo yum -y install docker +sudo usermod -a -G docker ec2-user +sudo service docker start +sudo chmod 666 /var/run/docker.sock +docker run --network host --rm -t -e CLOUD_LOCK={cloud_key} --name sssc stochss/stochss-compute:cloud stochss-compute-cluster -p {self._remote_config.api_port} > /home/ec2-user/sssc-out 2> /home/ec2-user/sssc-err & +''' + kwargs = { + 'ImageId': self._ami, + 'InstanceType': instance_type, + 'KeyName': self._remote_config.key_name, + 'MinCount': 1, + 'MaxCount': 1, + 'SubnetId': self._subnets['public'].id, + 'SecurityGroupIds': [self._default_security_group.id, self._server_security_group.id], + 'TagSpecifications': [ + { + 'ResourceType': 'instance', + 'Tags': [ + { + 'Key': 'Name', + 'Value': self._remote_config.server_name + }, + ] + }, + ], + 'UserData': launch_commands, + } + + log.info( + 'Launching StochSS-Compute server instance. This might take a minute.......') + try: + response = self._client.run_instances(**kwargs) + except ClientError as c_e: + raise EC2Exception from c_e + + instance_id = response['Instances'][0]['InstanceId'] + # try catch + self._server = self._resources.Instance(instance_id) + self._server.wait_until_exists() + self._server.wait_until_running() + + log.info('Instance "%s" is running.', instance_id) + + self._poll_launch_progress(['sssc']) + + log.info('Restricting server access to only your ip.') + source_ip = self._get_source_ip(cloud_key) + + self._restrict_ingress(source_ip) + self._init = True + log.info('StochSS-Compute ready to go!') + + def _poll_launch_progress(self, container_names, mock=False): + """ + Polls the instance to see if the Docker container is running. + + :param container_names: A list of Docker container names to check against. + :type container_names: List[str] + """ + if mock is True: + from test.unit_tests.mock_ssh import MockSSH + ssh = MockSSH() + else: + ssh = SSHClient() + ssh.set_missing_host_key_policy(AutoAddPolicy()) + sshtries = 0 + while True: + try: + ssh.connect(self._server.public_ip_address, username='ec2-user', + key_filename=self._local_config.key_path, look_for_keys=False) + break + except Exception as err2: + if sshtries >= 5: + raise err2 + self._server.reload() + sleep(5) + sshtries += 1 + continue + for container in container_names: + sshtries = 0 + while True: + sleep(60) + _, stdout, stderr = ssh.exec_command( + "docker container inspect -f '{{.State.Running}}' " + f'{container}') + rc = stdout.channel.recv_exit_status() + out = stdout.readlines() + err2 = stderr.readlines() + if rc == -1: + ssh.close() + raise EC2Exception( + "Something went wrong connecting to the server. No exit status provided by the server.") + # Wait for yum update, docker install, container download + if rc == 1 or rc == 127: + log.info('Waiting on Docker daemon.') + sshtries += 1 + if sshtries >= 5: + ssh.close() + raise EC2Exception( + f"Something went wrong with Docker. Max retry attempts exceeded.\nError:\n{''.join(err2)}") + if rc == 0: + if 'true\n' in out: + sleep(10) + log.info('Container "%s" is running.', container) + break + ssh.close() + + def _get_source_ip(self, cloud_key): + """ + Ping the server to find the IP address associated with the request. + + :param cloud_key: A secret key which must match the random key used to launch the instance. + :type cloud_key: str + """ + source_ip_request = SourceIpRequest(cloud_key=cloud_key) + response_raw = self.post( + Endpoint.CLOUD, sub='/sourceip', request=source_ip_request) + if not response_raw.ok: + raise EC2Exception(response_raw.reason) + response = SourceIpResponse.parse(response_raw.text) + return response.source_ip + + def _load_cluster(self): + ''' + Reload cluster resources. Returns False if no vpc named sssc-vpc. + ''' + + vpc_search_filter = [ + { + 'Name': 'tag:Name', + 'Values': [ + self._remote_config.vpc_name + ] + } + ] + vpc_response = self._client.describe_vpcs(Filters=vpc_search_filter) + + if len(vpc_response['Vpcs']) == 0: + if os.path.exists(self._local_config.key_path): + self._set_status('key error') + raise ResourceException + else: + try: + keypair = self._client.describe_key_pairs( + KeyNames=[self._remote_config.key_name]) + if keypair is not None: + self._set_status('key error') + raise ResourceException + except: + pass + return False + if len(vpc_response['Vpcs']) == 2: + log.error('More than one VPC named "%s".', + self._remote_config.vpc_name) + self._set_status('VPC error') + raise ResourceException + vpc_id = vpc_response['Vpcs'][0]['VpcId'] + self._vpc = self._resources.Vpc(vpc_id) + vpc = self._vpc + errors = False + for instance in vpc.instances.all(): + for tag in instance.tags: + if tag['Key'] == 'Name' and tag['Value'] == self._remote_config.server_name: + self._server = instance + if self._server is None: + log.warn('No instances named "%s".', + self._remote_config.server_name) + self._set_status('server error') + errors = True + for s_g in vpc.security_groups.all(): + if s_g.group_name == 'default': + self._default_security_group = s_g + if s_g.group_name == self._remote_config.security_group_name: + for rule in s_g.ip_permissions: + if rule['FromPort'] == self._remote_config.api_port \ + and rule['ToPort'] == self._remote_config.api_port \ + and rule['IpRanges'][0]['CidrIp'] == '0.0.0.0/0': + log.warn('Security Group rule error.') + self._set_status('security group error') + errors = True + self._server_security_group = s_g + if self._server_security_group is None: + log.warn('No security group named "%s".', + self._remote_config.security_group_name) + self._set_status('security group error') + errors = True + for subnet in vpc.subnets.all(): + for tag in subnet.tags: + if tag['Key'] == 'Name' and tag['Value'] == f'{self._remote_config.subnet_name}-public': + self._subnets['public'] = subnet + if tag['Key'] == 'Name' and tag['Value'] == f'{self._remote_config.subnet_name}-private': + self._subnets['private'] = subnet + if None in self._subnets.values(): + log.warn('Missing or misconfigured subnet.') + self._set_status('subnet error') + errors = True + if errors is True: + raise ResourceException + else: + self._init = True + log.info('Cluster loaded.') + self._set_status(self._server.state['Name']) + return True diff --git a/gillespy2/remote/cloud/ec2_config.py b/gillespy2/remote/cloud/ec2_config.py new file mode 100644 index 00000000..0fa59d26 --- /dev/null +++ b/gillespy2/remote/cloud/ec2_config.py @@ -0,0 +1,119 @@ +''' +gillespy2.remote.cloud.ec2_config +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os + +class EC2RemoteConfig: + ''' + Configure remote settings. + + :param suffix: Add a suffix to all AWS resource names. + :type suffix: str + + :param vpc_name: Identify the VPC. + :type vpc_name: str + + :param subnet_name: Identify the subnet. + :type subnet_name: str + + :param security_group_name: Identify the security group. + :type security_group_name: str + + :param server_name: Identify the server. + :type server_name: str + + :param key_name: Identify the AWS EC2 KeyPair. + :type key_name: str + + :param api_port: Port to serve from. + :type api_port: int + + :param region: Region to point to, like 'us-east-1' See `here `_. + :type region: str + + :param ami: Custom AMI to use, like 'ami-09d3b3274b6c5d4aa'. See `here `_. + :type ami: str + ''' + _AMIS = { + 'us-east-1': 'ami-09d3b3274b6c5d4aa', + 'us-east-2': 'ami-089a545a9ed9893b6', + 'us-west-1': 'ami-017c001a88dd93847', + 'us-west-2': 'ami-0d593311db5abb72b', + } + + def __init__(self, + suffix=None, + vpc_name='sssc-vpc', + subnet_name='sssc-subnet', + security_group_name='sssc-sg', + server_name='sssc-server', + key_name='sssc-server-ssh-key', + api_port=29681, + region=None, + ami=None, + ): + if suffix is not None: + suffix = f'-{suffix}' + else: + suffix = '' + + self.vpc_name = vpc_name + suffix + self.subnet_name = subnet_name + suffix + self.security_group_name = security_group_name + suffix + self.server_name = server_name + suffix + self.key_name = key_name + suffix + self.api_port = api_port + self.region = region + self.ami = ami + + +class EC2LocalConfig: + ''' + Configure local settings. + + :param key_dir: Path to a directory to store SSH key. + :type key_dir: str + + :param key_name: Name for the file. + :type key_name: str + + :param status_file: Path to a file to write instance status. Writes status to top line of that file. + :type status_file: str + + :param key_type: ed25519 or rsa + :type key_type: str + + :param key_format: pem or ppk + :type key_format: str + ''' + + def __init__(self, + key_dir='./.sssc', + key_name='sssc-server-ssh-key', + status_file=None, + key_type='ed25519', + key_format='pem', + ): + self.key_dir = key_dir + self._key_filename = f'{key_name}.{key_format}' + self.key_type = key_type + self.key_format = key_format + self.key_path = os.path.abspath( + os.path.join(self.key_dir, self._key_filename)) + self.status_file = status_file diff --git a/gillespy2/remote/cloud/exceptions.py b/gillespy2/remote/cloud/exceptions.py new file mode 100644 index 00000000..3bb21f84 --- /dev/null +++ b/gillespy2/remote/cloud/exceptions.py @@ -0,0 +1,41 @@ +''' +gillespy2.remote.cloud.exceptions +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +class ResourceException(Exception): + ''' + Misconfigured or out-of-date resources detected in the cluster setup. + ''' + def __init__(self, *args: object) -> None: + super().__init__(*args) + print('Missing or misconfigured resources.') + +class EC2ImportException(Exception): + ''' + Some extra dependencies are required for EC2. + ''' + def __init__(self, *args: object) -> None: + super().__init__(*args) + print('StochSS-Compute on EC2 requires boto3 and paramiko to be installed.\nTry: pip install boto3 paramiko') + +class EC2Exception(Exception): + ''' + General exception class. + ''' + def __init__(self, *args: object) -> None: + super().__init__(*args) diff --git a/gillespy2/remote/core/__init__.py b/gillespy2/remote/core/__init__.py new file mode 100644 index 00000000..22883bea --- /dev/null +++ b/gillespy2/remote/core/__init__.py @@ -0,0 +1,22 @@ +''' +gillespy2.remote.core +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .errors import RemoteSimulationError +from .remote_results import RemoteResults +from .remote_simulation import RemoteSimulation diff --git a/gillespy2/remote/core/errors.py b/gillespy2/remote/core/errors.py new file mode 100644 index 00000000..97e7f645 --- /dev/null +++ b/gillespy2/remote/core/errors.py @@ -0,0 +1,29 @@ +''' +gillespy2.remote.core.errors +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +class RemoteSimulationError(Exception): + ''' + General simulation error. + ''' + +class CacheError(Exception): + ''' + General cache error. + ''' + \ No newline at end of file diff --git a/gillespy2/remote/core/exceptions.py b/gillespy2/remote/core/exceptions.py new file mode 100644 index 00000000..c228ad57 --- /dev/null +++ b/gillespy2/remote/core/exceptions.py @@ -0,0 +1,33 @@ +''' +gillespy2.remote.core.exceptions +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +class PRNGCollision(Exception): + ''' + ...Lucky??? + ''' + +class MessageParseException(Exception): + ''' + Raised if there is an error parsing a raw HTTP message. + ''' + +class StatusException(Exception): + ''' + Raised if there is an error determining the status of a simulation. + ''' \ No newline at end of file diff --git a/gillespy2/remote/core/messages/__init__.py b/gillespy2/remote/core/messages/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gillespy2/remote/core/messages/base.py b/gillespy2/remote/core/messages/base.py new file mode 100644 index 00000000..b3d85a6f --- /dev/null +++ b/gillespy2/remote/core/messages/base.py @@ -0,0 +1,52 @@ +''' +gillespy2.remote.core.messages +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from abc import ABC, abstractmethod + +class Request(ABC): + ''' + Base class. + ''' + @abstractmethod + def encode(self): + ''' + Encode self for http. + ''' + @staticmethod + @abstractmethod + def parse(raw_request): + ''' + Parse http for python. + ''' + +class Response(ABC): + ''' + Base class. + ''' + @abstractmethod + def encode(self): + ''' + Encode self for http. + ''' + @staticmethod + @abstractmethod + def parse(raw_response): + ''' + Parse http for python. + ''' diff --git a/gillespy2/remote/core/messages/results.py b/gillespy2/remote/core/messages/results.py new file mode 100644 index 00000000..dc657c1e --- /dev/null +++ b/gillespy2/remote/core/messages/results.py @@ -0,0 +1,83 @@ +''' +gillespy2.remote.core.messages.results +''' +from tornado.escape import json_decode +from gillespy2 import Results + +from gillespy2.remote.core.messages.base import Request, Response + +class ResultsRequest(Request): + ''' + Request results from the server. + + :param results_id: Hash of the SimulationRunCacheRequest + :type results_id: str + + :param namespace: Optional namespace to prepend to results directory. + :type namespace: str + + ''' + def __init__(self, results_id, namespace=None, n_traj=None): + self.results_id = results_id + self.namespace = namespace + self.n_traj = n_traj + + def encode(self): + ''' + :returns: self.__dict__ + :rtype: dict + ''' + return self.__dict__ + + @staticmethod + def parse(raw_request): + ''' + Parse HTTP request. + + :param raw_request: The request. + :type raw_request: dict[str, str] + + :returns: The decoded object. + :rtype: ResultsRequest + ''' + request_dict = json_decode(raw_request) + results_id = request_dict.get('results_id', None) + namespace = request_dict.get('namespace', None) + n_traj = request_dict.get('n_traj', None) + return ResultsRequest(results_id, namespace, n_traj) + +class ResultsResponse(Response): + ''' + A response from the server about the Results. + + :param results: The requested Results from the cache. (JSON) + :type results: str + + ''' + def __init__(self, results = None): + self.results = results + + def encode(self): + ''' + :returns: self.__dict__ + :rtype: dict + ''' + return {'results': self.results or ''} + + @staticmethod + def parse(raw_response): + ''' + Parse HTTP response. + + :param raw_response: The response. + :type raw_response: dict[str, str] + + :returns: The decoded object. + :rtype: ResultsResponse + ''' + response_dict = json_decode(raw_response) + if response_dict['results'] != '': + results = Results.from_json(response_dict['results']) + else: + results = None + return ResultsResponse(results) \ No newline at end of file diff --git a/gillespy2/remote/core/messages/simulation_run_cache.py b/gillespy2/remote/core/messages/simulation_run_cache.py new file mode 100644 index 00000000..45419ab8 --- /dev/null +++ b/gillespy2/remote/core/messages/simulation_run_cache.py @@ -0,0 +1,184 @@ +''' +gillespy2.remote.core.messages.simulation_run_cache +''' +from hashlib import md5 +from json import JSONDecodeError +from secrets import token_hex +from tornado.escape import json_decode, json_encode +from gillespy2 import Results +from gillespy2.core.model import Model +from gillespy2.remote.core.exceptions import MessageParseException +from gillespy2.remote.core.messages.base import Request, Response +from gillespy2.remote.core.messages.status import SimStatus + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class SimulationRunCacheRequest(Request): + ''' + A simulation request message object. + + :param model: A model to run. + :type model: gillespy2.Model + + :param namespace: Optional namespace for the results. + :type namespace: str | None + + :param kwargs: kwargs for the model.run() call. + :type kwargs: dict[str, Any] + ''' + def __init__(self, + model, + namespace=None, + force_run=False, + ignore_cache=False, + parallelize=False, + chunk_trajectories=False, + **kwargs): + self.model = model + self.kwargs = kwargs + self.id = token_hex(16) + if ignore_cache is True: + self.results_id = self.id + else: + self.results_id = self.hash() + self.namespace = namespace + self.force_run = force_run + self.ignore_cache = ignore_cache + self.parallelize = parallelize + self.chunk_trajectories = chunk_trajectories + + def encode(self): + ''' + JSON-encode model and then encode self to dict. + ''' + return { + 'model': self.model.to_json(), + 'kwargs': self.kwargs, + 'id': self.id, + 'results_id': self.results_id, + 'namespace': self.namespace, + 'force_run': self.force_run, + 'ignore_cache': self.ignore_cache, + 'parallelize': self.parallelize, + 'chunk_trajectories': self.chunk_trajectories, + } + + @staticmethod + def parse(raw_request): + ''' + Parse raw HTTP request. Done server-side. + + :param raw_request: The request. + :type raw_request: dict[str, str] + + :returns: The decoded object. + :rtype: SimulationRunRequest + ''' + try: + request_dict = json_decode(raw_request) + except JSONDecodeError as err: + raise MessageParseException from err + + model = Model.from_json(request_dict.get('model', None)) + kwargs = request_dict.get('kwargs', {}) + id = request_dict.get('id', None) # apply correct token (from raw request) after object construction. + results_id = request_dict.get('results_id', None) # apply correct token (from raw request) after object construction. + namespace = request_dict.get('namespace', None) + force_run = request_dict.get('force_run', None) + ignore_cache = request_dict.get('ignore_cache', None) + parallelize = request_dict.get('parallelize', None) + chunk_trajectories = request_dict.get('chunk_trajectories', None) + if None in (model, id, results_id, force_run, ignore_cache, parallelize, chunk_trajectories): + raise MessageParseException + _ = SimulationRunCacheRequest(model, + namespace=namespace, + force_run=force_run, + ignore_cache=ignore_cache, + parallelize=parallelize, + chunk_trajectories=chunk_trajectories, + **kwargs) + _.id = id # apply correct token (from raw request) after object construction. + _.results_id = results_id # apply correct token (from raw request) after object construction. + return _ + + def hash(self): + ''' + Generate a unique hash of this simulation request. + Does not include number_of_trajectories in this calculation. + + :returns: md5 hex digest. + :rtype: str + ''' + log.debug('SimulationRunCacheRequest.hash()...') + anon_model_string = self.model.to_anon().to_json(encode_private=False) + popped_kwargs = {kw:self.kwargs[kw] for kw in self.kwargs if kw!='number_of_trajectories'} + # Explanation of line above: + # Take 'self.kwargs' (a dict), and add all entries to a new dictionary, + # EXCEPT the 'number_of_trajectories' key/value pair. + kwargs_string = json_encode(popped_kwargs) + log.debug('kwargs_string:') + log.debug(kwargs_string) + request_string = f'{anon_model_string}{kwargs_string}' + _hash = md5(str.encode(request_string)).hexdigest() + log.debug('_hash:') + log.debug(_hash) + return _hash + + +class SimulationRunCacheResponse(Response): + ''' + A response from the server regarding a SimulationRunCacheRequest. + + :param status: The status of the simulation. + :type status: SimStatus + + :param error_message: Possible error message. + :type error_message: str | None + + :param results_id: Hash of the simulation request. Identifies the results. + :type results_id: str | None + + :param task_id: Task ID. Handle to a running simulation. + :type task_id: str | None + + :param results: JSON-Encoded gillespy2.Results + :type results: str | None + ''' + def __init__(self, status, error_message = None, results_id = None, results = None, task_id = None): + self.status = status + self.error_message = error_message + self.results_id = results_id + self.results = results + self.task_id = task_id + + def encode(self): + ''' + Encode self to dict. + ''' + return {'status': self.status.name, + 'error_message': self.error_message, + 'results_id': self.results_id, + 'results': self.results, + 'task_id': self.task_id,} + + @staticmethod + def parse(raw_response): + ''' + Parse HTTP response. + + :param raw_response: The response. + :type raw_response: dict[str, str] + + :returns: The decoded object. + :rtype: SimulationRunResponse + ''' + response_dict = json_decode(raw_response) + status = SimStatus.from_str(response_dict['status']) + results_id = response_dict.get('results_id', None) + error_message = response_dict.get('error_message', None) + task_id = response_dict.get('task_id', None) + results = response_dict.get('results', None) + if results is not None: + results = Results.from_json(results) + return SimulationRunCacheResponse(status, error_message, results_id, results, task_id) diff --git a/gillespy2/remote/core/messages/source_ip.py b/gillespy2/remote/core/messages/source_ip.py new file mode 100644 index 00000000..85819738 --- /dev/null +++ b/gillespy2/remote/core/messages/source_ip.py @@ -0,0 +1,68 @@ +''' +gillespy2.remote.core.messages.source_ip +''' +from tornado.escape import json_decode +from gillespy2.remote.core.messages.base import Request, Response + +class SourceIpRequest(Request): + ''' + Restrict server access. + + :param cloud_key: Random key generated locally during launch. + :type cloud_key: str + ''' + + def __init__(self, cloud_key): + self.cloud_key = cloud_key + + def encode(self): + ''' + :returns: self.__dict__ + :rtype: dict + ''' + return self.__dict__ + + @staticmethod + def parse(raw_request): + ''' + Parse HTTP request. + + :param raw_request: The request. + :type raw_request: dict[str, str] + + :returns: The decoded object. + :rtype: SourceIpRequest + ''' + request_dict = json_decode(raw_request) + return SourceIpRequest(request_dict['cloud_key']) + +class SourceIpResponse(Response): + ''' + Response from server containing IP address of the source. + + :param source_ip: IP address of the client. + :type source_ip: str + ''' + def __init__(self, source_ip): + self.source_ip = source_ip + + def encode(self): + ''' + :returns: self.__dict__ + :rtype: dict + ''' + return self.__dict__ + + @staticmethod + def parse(raw_response): + ''' + Parses a http response and returns a python object. + + :param raw_response: A raw http SourceIpResponse from the server. + :type raw_response: str + + :returns: The decoded object. + :rtype: SourceIpResponse + ''' + response_dict = json_decode(raw_response) + return SourceIpResponse(response_dict['source_ip']) diff --git a/gillespy2/remote/core/messages/status.py b/gillespy2/remote/core/messages/status.py new file mode 100644 index 00000000..8cb0f581 --- /dev/null +++ b/gillespy2/remote/core/messages/status.py @@ -0,0 +1,125 @@ +''' +gillespy2.remote.core.messages.status +''' +from enum import Enum +from tornado.escape import json_decode +from gillespy2.remote.core.exceptions import MessageParseException +from gillespy2.remote.core.messages.base import Request, Response + +class SimStatus(Enum): + ''' + Status describing a remote simulation. + ''' + RUNNING = 'The simulation is currently running.' + READY = 'Simulation is done and results exist in the cache.' + ERROR = 'The Simulation has encountered an error.' + DOES_NOT_EXIST = 'There is no evidence of this simulation either running or on disk.' + + @staticmethod + def from_str(name): + ''' + Convert str to Enum. + ''' + if name == 'RUNNING': + return SimStatus.RUNNING + if name == 'READY': + return SimStatus.READY + if name == 'ERROR': + return SimStatus.ERROR + if name == 'DOES_NOT_EXIST': + return SimStatus.DOES_NOT_EXIST + # pylint: disable=no-member + raise ValueError(f'Not a valid status.\n{SimStatus._member_names_}') + # pylint: enable=no-member + +class StatusRequest(Request): + ''' + A request for simulation status. + + :param results_id: Hash of the SimulationRunCacheRequest or key from SimulationRunRequest + :type results_id: str + + :param n_traj: Number of requested trajectories. Defaults to 1. + :type n_traj: int | None + + :param task_id: Handle to a currently running simulation. + :type task_id: str | None + + :param namespace: Optional namespace to prepend to results directory. + :type namespace: str | None + ''' + def __init__(self, results_id, n_traj=None, task_id=None, namespace=None): + self.results_id = results_id + self.namespace = namespace + self.n_traj = n_traj + self.task_id = task_id + + def encode(self): + ''' + :returns: self.__dict__ + :rtype: dict + ''' + return self.__dict__ + + @staticmethod + def parse(raw_request): + ''' + Parse HTTP request. + + :param raw_request: The request. + :type raw_request: dict[str, str] + + :returns: The decoded object. + :rtype: StatusRequest + ''' + request_dict = json_decode(raw_request) + results_id = request_dict.get('results_id', None) + namespace = request_dict.get('namespace', None) + n_traj = request_dict.get('n_traj', None) + task_id = request_dict.get('task_id', None) + return StatusRequest(results_id, namespace, n_traj, task_id) + +class StatusResponse(Response): + ''' + A response from the server about simulation status. + + :param status: Status of the simulation + :type status: SimStatus + + :param message: Possible error message or otherwise + :type message: str | None + ''' + def __init__(self, status, message=None): + self.status = status + self.message = message + + def encode(self): + ''' + Encodes self. + :returns: self as dict + :rtype: dict + ''' + return {'status': self.status.name, + 'message': self.message} + + @staticmethod + def parse(raw_response): + ''' + Parse HTTP response. + + :param raw_response: The response. + :type raw_response: dict[str, str] + + :returns: The decoded object. + :rtype: StatusResponse + ''' + + response_dict = json_decode(raw_response) + + try: + status = SimStatus.from_str(response_dict.get('status')) + except ValueError as err: + raise MessageParseException from err + + message = response_dict.get('message', None) + return StatusResponse(status, message=message) \ No newline at end of file diff --git a/gillespy2/remote/core/remote_results.py b/gillespy2/remote/core/remote_results.py new file mode 100644 index 00000000..702002fb --- /dev/null +++ b/gillespy2/remote/core/remote_results.py @@ -0,0 +1,169 @@ +''' +gillespy2.remote.core.remote_results +''' +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from time import sleep +from gillespy2 import Results +from gillespy2.remote.client.endpoint import Endpoint +from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.exceptions import StatusException +from gillespy2.remote.core.messages.results import ResultsRequest, ResultsResponse +from gillespy2.remote.core.messages.status import StatusRequest, StatusResponse, SimStatus + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class RemoteResults(Results): + ''' + Wrapper for a gillespy2.Results object that exists on a remote server and which is then downloaded locally. + A Results object is: A List of Trajectory objects created by a gillespy2 solver, extends the UserList object. + + These four fields must be initialized manually: id, server, n_traj, task_id. + + :param data: A list of trajectory objects. + :type data: UserList + + :param id: ID of the cached Results object. + :type id: str + + :param server: The remote instance of StochSS-Compute where the Results are cached. + :type server: gillespy2.remote.ComputeServer + + :param task_id: Handle for the running simulation. + :type task_id: str + + :param namespace: Optional namespace. + :type namespace: str + ''' + + id = None # required + server = None + n_traj = None # Defaults to 1 + task_id = None # optional + namespace = None # optional + + # pylint:disable=super-init-not-called + def __init__(self, data = None): + self._data = data + # pylint:enable=super-init-not-called + + @staticmethod + def factory(id, server, n_traj, task_id=None, namespace=None, data=None): + remote_results = RemoteResults(data=data) + remote_results.id = id + remote_results.task_id = task_id + remote_results.server = server + remote_results.n_traj = n_traj + remote_results.namespace = namespace + return remote_results + + @property + def data(self): + """ + The trajectory data. + + :returns: self._data + :rtype: UserList + """ + if None in (self.id, self.server, self.n_traj): + raise Exception('RemoteResults must have a self.id, self.server and self.n_traj.') + + if self._data is None: + self._resolve() + return self._data + + @property + def sim_status(self): + ''' + Fetch the simulation status. + + :returns: Simulation status enum as a string. + :rtype: str + ''' + if self._data is None: + return self._status().status.name + else: + raise StatusException('Cannot call status on a finished simulation.') + + def get_gillespy2_results(self): + """ + Get the GillesPy2 results object from the remote results. + + :returns: The generated GillesPy2 results object. + :rtype: gillespy.Results + """ + return Results(self.data) + + + @property + def is_ready(self): + """ + True if results exist in cache on the server. + + :returns: status == SimStatus.READY + :rtype: bool + """ + if self._data is not None: + return self._status().status == SimStatus.READY + return True + + def _status(self): + ''' + It is undefined/illegal behavior to call this function if self._data is not None. + ''' + if self._data is not None: + raise StatusException('Cannot call status on a finished simulation.') + + status_request = StatusRequest(self.id, self.n_traj, self.task_id, self.namespace) + + response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, '/status', request=status_request) + + if not response_raw.ok: + raise RemoteSimulationError(response_raw.reason) + + status_response = StatusResponse.parse(response_raw.text) + return status_response + + def _resolve(self): + ''' + It is undefined behavior to call this function if self._data is not None. + ''' + status_response = self._status() + status = status_response.status + + if status == SimStatus.RUNNING: + log.info('Simulation is running. Downloading results when complete......') + while True: + sleep(5) + status_response = self._status() + status = status_response.status + if status != SimStatus.RUNNING: + break + + if status in (SimStatus.DOES_NOT_EXIST, SimStatus.ERROR): + raise RemoteSimulationError(status_response.message) + + if status == SimStatus.READY: + log.info('Results ready. Fetching.......') + results_request = ResultsRequest(self.id, self.namespace, self.n_traj) + response_raw = self.server.get(Endpoint.SIMULATION_GILLESPY2, f"/results", request=results_request) + if not response_raw.ok: + raise RemoteSimulationError(response_raw.reason) + + response = ResultsResponse.parse(response_raw.text) + + self._data = response.results.data # Fully initialized diff --git a/gillespy2/remote/core/remote_simulation.py b/gillespy2/remote/core/remote_simulation.py new file mode 100644 index 00000000..a28aa26b --- /dev/null +++ b/gillespy2/remote/core/remote_simulation.py @@ -0,0 +1,252 @@ +''' +gillespy2.remote.core.remote_simulation +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from gillespy2.remote.client.endpoint import Endpoint +from gillespy2.remote.core.messages.simulation_run_cache import SimulationRunCacheRequest, SimulationRunCacheResponse +from gillespy2.remote.core.messages.status import SimStatus, StatusRequest +from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.remote_results import RemoteResults + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class RemoteSimulation: + ''' + An object representing a remote gillespy2 simulation. Requires a model and a host address. + A solver type may be provided, but does not accept instantiated solvers. + + :param model: The model to simulate. + :type model: gillespy2.Model + + :param server: A server to run the simulation. Optional if host is provided. + :type server: gillespy2.remote.Server + + :param host: The address of a running instance of gillespy2.remote. Optional if server is provided. + :type host: str + + :param port: The port to use when connecting to the host. + Only needed if default server port is changed. Defaults to 29681. + :type port: int + + :param solver: The type of solver to use. Does not accept instantiated solvers. + :type solver: gillespy2.GillesPySolver + ''' + def __init__(self, + model, + server = None, + host: str = None, + port: int = 29681, + solver = None, + ): + + if server is not None and host is not None: + raise RemoteSimulationError('Pass a ComputeServer/Cluster object or host but not both.') + if server is None and host is None: + raise RemoteSimulationError('Pass a ComputeServer/Cluster object or host.') + if server is None and port is None: + raise RemoteSimulationError('Pass a ComputeServer/Cluster object or port.') + + if server is None: + from gillespy2.remote.client.compute_server import ComputeServer + self.server = ComputeServer(host, port) + else: + self.server = server + + self.model = model + + if solver is not None: + if hasattr(solver, 'is_instantiated'): + raise RemoteSimulationError( + 'RemoteSimulation does not accept an instantiated solver object. Pass a type.') + self.solver = solver + + def is_cached(self, namespace=None, **params): + ''' + Checks to see if a dummy simulation exists in the cache. + + :param namespace: If provided, prepend to results path. + :type namespace: str + + :param params: Arguments for simulation. + :type params: dict[str, Any] + + :returns: If the results are cached on the server. + :rtype: bool + ''' + log.debug('is_cached()') + n_traj_requested = params.get('number_of_trajectories', 1) + n_traj_cached = self.get_n_traj_in_cache(namespace=namespace, + **params) + if n_traj_requested <= n_traj_cached: + return True + else: + return False + + def run(self, namespace=None, ignore_cache=False, parallelize=False, chunk_trajectories=False, **params): + # pylint:disable=line-too-long + """ + Simulate the Model on the target ComputeServer, returning the results or a handle to a running simulation. + See `here `_. + + :param namespace: If provided, prepend to results path. + :type namespace: str + + :param ignore_cache: When True, ignore cache completely and return always new results. + :type ignore_cache: bool + + :param params: Arguments to pass directly to the Model#run call on the server. + :type params: dict[str, Any] + + :returns: RemoteResults populated with Results if cached, otherwise and unpopulated RemoteResults + :rtype: RemoteResults + + :raises RemoteSimulationError: In the case of SimStatus.ERROR + """ + # pylint:enable=line-too-long + + params = self._encode_solver_arg(**params) + sim_request = SimulationRunCacheRequest(self.model, + namespace=namespace, + ignore_cache=ignore_cache, + parallelize=parallelize, + chunk_trajectories=chunk_trajectories, + **params) + log.debug('run()...') + log.debug('sim_request.results_id') + log.debug(sim_request.results_id) + return self._run_cache(sim_request) + + def _run_cache(self, request): + ''' + :param request: Request to send to the server. Contains Model and related arguments. + :type request: SimulationRunRequest + ''' + log.debug('_run_cache(request)...') + log.debug('request.kwargs.get("solver", None)') + log.debug(request.kwargs.get('solver', None)) + response_raw = self.server.post(Endpoint.SIMULATION_GILLESPY2, sub='/run/cache', request=request) + + if not response_raw.ok: + raise Exception(response_raw.reason) + + sim_response = SimulationRunCacheResponse.parse(response_raw.text) + + if sim_response.status == SimStatus.ERROR: + raise RemoteSimulationError(sim_response.error_message) + if sim_response.status == SimStatus.READY: + data = sim_response.results.data + else: + data = None + return RemoteResults.factory(request.results_id, + self.server, + request.kwargs.get('number_of_trajectories', 1), + request.id, + request.namespace, + data) + + def run_distributed(self, namespace=None, force_run=False, **params): + log.debug('run_distributed()...') + log.debug('namespace:') + log.debug(namespace) + log.debug('force_run:') + log.debug(force_run) + log.debug('params.get("solver", None)') + log.debug(params.get('solver', None)) + params = self._encode_solver_arg(**params) + log.debug('params.get("solver", None)') + log.debug(params.get('solver', None)) + response_raw = self.server.get(Endpoint.DASK, sub='/number_of_workers') + n_workers = int(response_raw.text) + log.debug('n_workers:') + log.debug(n_workers) + n_traj_requested = params.get('number_of_trajectories', 1) + results_collection = [] + if force_run is True: + n_traj = n_traj_requested + if force_run is False: + n_traj_remote = self.get_n_traj_in_cache(namespace=namespace, + **params) + log.debug('n_traj_remote:') + log.debug(n_traj_remote) + n_traj = n_traj_requested - n_traj_remote + + traj_per_worker = n_traj // n_workers + log.debug('traj_per_worker:') + log.debug(traj_per_worker) + extra_traj = n_traj % n_workers + log.debug('extra_traj:') + log.debug(extra_traj) + if traj_per_worker > 0: + for _ in range(n_workers): + params['number_of_trajectories'] = traj_per_worker + sim_request = SimulationRunCacheRequest(self.model, + namespace=namespace, + force_run=True, + **params) + results_collection.append(self._run_cache(sim_request)) + if extra_traj > 0: + params['number_of_trajectories'] = extra_traj + sim_request = SimulationRunCacheRequest(self.model, + namespace=namespace, + force_run=True, + **params) + results_collection.append(self._run_cache(sim_request)) + + return RemoteResults.factory(sim_request.results_id, + self.server, + n_traj_requested, + None, + sim_request.namespace, + None) + + def get_n_traj_in_cache(self, namespace=None, **params) -> int: + log.debug('get_n_traj_in_cache()...') + log.debug('namespace:') + log.debug(namespace) + params = self._encode_solver_arg(**params) + simulation_request = SimulationRunCacheRequest(self.model, + namespace=namespace, + **params) + log.debug('simulation_request.results_id:') + log.debug(simulation_request.results_id) + n_traj = params.get('number_of_trajectories', 1) + status_request = StatusRequest(results_id=simulation_request.results_id, + n_traj=n_traj, + namespace=namespace) + response_raw = self.server.get(Endpoint.CACHE, sub='/number_of_trajectories', request=status_request) + n_traj_in_cache = int(response_raw.text) + log.debug('n_traj_in_cache:') + log.debug(n_traj_in_cache) + return n_traj_in_cache + + def _encode_solver_arg(self, **params): + log.debug('_encode_solver_arg()...') + value = params.get('solver', '') # Makes for a better fail if they pass something invalid. + log.debug('params.get("solver", None)') + log.debug(params.get('solver', None)) + if type(value) is not str: + if hasattr(params['solver'], 'is_instantiated'): + raise RemoteSimulationError( + 'RemoteSimulation does not accept an instantiated solver object. Pass a type.') + params['solver'] = f"{value.__module__}.{value.__qualname__}" + if self.solver is not None and value == '': + params["solver"] = f"{self.solver.__module__}.{self.solver.__qualname__}" + log.debug('params.get("solver", None)') + log.debug(params.get('solver', None)) + return params diff --git a/gillespy2/remote/core/utils/__init__.py b/gillespy2/remote/core/utils/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/gillespy2/remote/core/utils/dask.py b/gillespy2/remote/core/utils/dask.py new file mode 100644 index 00000000..e9dc3eba --- /dev/null +++ b/gillespy2/remote/core/utils/dask.py @@ -0,0 +1,25 @@ +import os +message = '''distributed: + version: 2 + logging: + distributed: error + distributed.client: error + distributed.worker: error + distributed.nanny: error + distributed.core: error + ''' +def silence_dask_logs(): + config_path_string = '~/.config/dask/distributed.yaml' + config_path = os.path.expanduser(config_path_string) + if os.path.exists(config_path) is True: + print(f'To silence dask logs, edit the file {config_path} to contain:') + print() + print(message) + else: + config_dir_string = '~/.config/dask/' + config_dir = os.path.expanduser(config_dir_string) + if os.path.exists(config_dir) is False: + os.makedirs(config_dir) + with open(config_path, 'x', encoding='utf-8') as file: + file.write(message) + file.close() diff --git a/gillespy2/remote/core/utils/debug.py b/gillespy2/remote/core/utils/debug.py new file mode 100644 index 00000000..4cae165b --- /dev/null +++ b/gillespy2/remote/core/utils/debug.py @@ -0,0 +1,23 @@ +''' +gillespy2.remote.core.debug +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from logging import getLogger +getLogger().setLevel('DEBUG') +# Call the line below in a python process to initiate client-side logging +# import gillespy2.remote.core.utils.debug \ No newline at end of file diff --git a/gillespy2/remote/core/utils/log_config.py b/gillespy2/remote/core/utils/log_config.py new file mode 100644 index 00000000..506d0243 --- /dev/null +++ b/gillespy2/remote/core/utils/log_config.py @@ -0,0 +1,36 @@ +''' +gillespy2.remote.core.utils.log_config + +Global Logging Configuration +''' + +from logging import getLogger + +def init_logging(name): + ''' + Call after import to initialize logs in a module file. + To follow convention, use predefined __name__. + + Like so: + + from gillespy2.remote.core.utils.log_config import init_logging + log = init_logging(__name__) + + :param name: Name for the logger. Use the dot-separated module path string. + :type name: str + + :returns: A module specific logger. + :rtype: logging.Logger + ''' + logger = getLogger(name) + return logger + +def set_global_log_level(level): + ''' + Sets the root logger log level. + + :param level: NOTSET:0, DEBUG:10, INFO:20, WARNING:30, ERROR:40, CRITICAL:50, etc. + :type level: int | logging._Level + ''' + getLogger(__name__.split('.', maxsplit=1)[0]).setLevel(level) + \ No newline at end of file diff --git a/gillespy2/remote/launch.py b/gillespy2/remote/launch.py new file mode 100644 index 00000000..6887ee22 --- /dev/null +++ b/gillespy2/remote/launch.py @@ -0,0 +1,165 @@ +''' +gillespy2.remote.launch +''' +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import sys +import asyncio +from logging import INFO, getLevelName +from argparse import ArgumentParser, Namespace +from distributed import LocalCluster +from gillespy2.remote.server.api import start_api +from gillespy2.remote.core.utils.dask import silence_dask_logs +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +def _add_shared_args(parser): + ''' + :type parser: ArgumentParser + + :rtype: ArgumentParser + ''' + + server = parser.add_argument_group('Server') + server.add_argument("-p", "--port", default=29681, type=int, required=False, + help="The port to use for the server. Defaults to 29681.") + server.add_argument("-l", "--logging-level", default=INFO, required=False, + help='Set the logging level threshold. Str or int. Defaults to INFO (20).') + + cache = parser.add_argument_group('Cache') + cache.add_argument('-c', '--cache_path', default='cache/', required=False, + help='Path to use for the cache.') + cache.add_argument('--rm', default=False, action='store_true', required=False, + help='Whether to delete the cache upon exit. Default False.') + + return parser + + +def launch_server(): + ''' + Start the REST API. Alias to script "gillespy2-remote". + + `gillespy2-remote` + OR + `python -m gillespy2.remote.launch` + ''' + def _parse_args() -> Namespace: + desc = ''' + GillesPy2 Remote allows you to run simulations remotely on your own Dask cluster. + To launch both simultaneously, use `gillespy2-remote-cluster` instead. + Trajectories are automatically cached to support multiple users running the same model. + ''' + parser = ArgumentParser(description=desc, add_help=True, conflict_handler='resolve') + + parser = _add_shared_args(parser) + + dask = parser.add_argument_group('Dask') + dask.add_argument("-H", "--dask-host", default='localhost', required=False, + help="The host to use for the dask scheduler. Defaults to localhost.") + dask.add_argument("-P", "--dask-scheduler-port", default=8786, type=int, required=False, + help="The port to use for the dask scheduler. Defaults to 8786.") + return parser.parse_args() + + args = _parse_args() + args.logging_level = getLevelName(args.logging_level) + + asyncio.run(start_api(**args.__dict__)) + + +def launch_with_cluster(): + ''' + Start up a Dask cluster along with gillespy2.remote REST API. Alias to script "gillespy2-remote-cluster". + + `gillespy2-remote-cluster` + OR + `python -m gillespy2.remote.launch cluster` + ''' + + def _parse_args() -> Namespace: + desc = ''' + Startup script for a GillesPy2 Remote and pre-configured Dask Distributed cluster. + Command-line options allow you to override automatic cluster configuration. + Your trajectories are automatically cached to support multiple users running the same model. + ''' + parser = ArgumentParser(description=desc, add_help=True, conflict_handler='resolve') + + parser = _add_shared_args(parser) + + dask = parser.add_argument_group('Dask') + dask.add_argument("-H", "--dask-host", default=None, required=False, + help="The host to use for the dask scheduler. Defaults to localhost.") + dask.add_argument("-P", "--dask-scheduler-port", default=0, type=int, required=False, + help="The port to use for the dask scheduler. 0 for a random port. \ + Defaults to a random port.") + dask.add_argument('-W', '--dask-n-workers', default=None, type=int, required=False, + help='Configure the number of workers. Defaults to one per core.') + dask.add_argument('-T', '--dask-threads-per-worker', default=None, required=False, type=int, + help='Configure the threads per worker. \ + Default will let Dask decide based on your CPU.') + dask.add_argument('--dask-processes', default=None, required=False, type=bool, + help='Whether to use processes (True) or threads (False). \ + Defaults to True, unless worker_class=Worker, in which case it defaults to False.') # Keep as non-flag b/c of side effects. + dask.add_argument('-D', '--dask-dashboard-address', default=':8787', required=False, + help='Address on which to listen for the Bokeh diagnostics server \ + like ‘localhost:8787’ or ‘0.0.0.0:8787’. Defaults to ‘:8787’. \ + Set to None to disable the dashboard. Use ‘:0’ for a random port.') + dask.add_argument('-N', '--dask-name', default=None, required=False, + help='A name to use when printing out the cluster, defaults to the type name.') + dask.add_argument('--silence-dask', default=False, action='store_true', required=False, + help='Enable this flag to silence dask INFO log output.') + + return parser.parse_args() + + + args = _parse_args() + args.logging_level = getLevelName(args.logging_level) + if args.silence_dask is True: + silence_dask_logs() + dask_args = {} + for (arg, value) in vars(args).items(): + if arg.startswith('dask_'): + dask_args[arg[5:]] = value + log.info('Launching Dask Cluster.') + cluster = LocalCluster(**dask_args) + tokens = cluster.scheduler_address.split(':') + dask_host = tokens[1][2:] + dask_port = int(tokens[2]) + msg = f'Scheduler Address: <{cluster.scheduler_address}>' + log.info(msg) + for i, worker in cluster.workers.items(): + msg = f'Worker {i}: {worker}' + log.info(msg) + + msg = f'Dashboard Link: <{cluster.dashboard_link}>\n' + log.info(msg) + + try: + asyncio.run(start_api(port=args.port, cache_path=args.cache_path, + dask_host=dask_host, dask_scheduler_port=dask_port, rm=args.rm, logging_level=args.logging_level)) + except asyncio.exceptions.CancelledError: + pass + finally: + log.info('Shutting down cluster.') + asyncio.run(cluster.close()) + log.info('Cluster terminated.') + + +if __name__ == '__main__': + if len(sys.argv) > 1: + if sys.argv[1] == 'cluster': + del sys.argv[1] + launch_with_cluster() + launch_server() diff --git a/gillespy2/remote/server/__init__.py b/gillespy2/remote/server/__init__.py new file mode 100644 index 00000000..d9f77f5d --- /dev/null +++ b/gillespy2/remote/server/__init__.py @@ -0,0 +1,20 @@ +''' +gillespy2.remote.server +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from .api import start_api diff --git a/gillespy2/remote/server/api.py b/gillespy2/remote/server/api.py new file mode 100644 index 00000000..d40fee2d --- /dev/null +++ b/gillespy2/remote/server/api.py @@ -0,0 +1,109 @@ +''' +gillespy2.remote.server.api +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import asyncio +import subprocess +from logging import INFO +from tornado.web import Application +from gillespy2.remote.server.handlers.number_of_trajectories import NumberOfTrajectoriesHandler +from gillespy2.remote.server.handlers.number_of_workers import NumberOfWorkersHandler +from gillespy2.remote.server.handlers.simulation_run_cache import SimulationRunCacheHandler +from gillespy2.remote.server.handlers.sourceip import SourceIpHandler +from gillespy2.remote.server.handlers.status import StatusHandler +from gillespy2.remote.server.handlers.results import ResultsHandler +from gillespy2.remote.core.utils.log_config import init_logging, set_global_log_level +log = init_logging(__name__) + +def _make_app(dask_host, dask_scheduler_port, cache): + scheduler_address = f'{dask_host}:{dask_scheduler_port}' + scheduler_arg = {'scheduler_address': scheduler_address} + cache_arg = {'cache_dir': cache} + args1 = scheduler_arg | cache_arg + return Application([ + (r'/api/v3/simulation/gillespy2/run/cache', + SimulationRunCacheHandler, + args1), + (r'/api/v3/simulation/gillespy2/status', + StatusHandler, + args1), + (r'/api/v3/simulation/gillespy2/results', + ResultsHandler, + cache_arg), + (r'/api/v3/cloud/sourceip', SourceIpHandler), + (r'/api/v3/dask/number_of_workers', NumberOfWorkersHandler, scheduler_arg), + (r'/api/v3/cache/number_of_trajectories', NumberOfTrajectoriesHandler, cache_arg), + ]) + +async def start_api( + port = 29681, + cache_path = 'cache/', + dask_host = 'localhost', + dask_scheduler_port = 8786, + rm = False, + logging_level = INFO, + ): + """ + Start the REST API with the following arguments. + + :param port: The port to listen on. + :type port: int + + :param cache_trajectories: If True, default behavior is to cache trajectories. If False, trajectory cacheing is turned off by default. Can be overridden on client side. + :type cache_trajectories: bool + + :param cache_path: The cache directory path. Do not begin with /. + :type cache_path: str + + :param dask_host: The address of the dask cluster. + :type dask_host: str + + :param dask_scheduler_port: The port of the dask cluster. + :type dask_scheduler_port: int + + :param rm: Delete the cache when exiting this program. + :type rm: bool + + :param logging_level: Set log level for gillespy2.remote. + :type debug: logging._Level + """ + + set_global_log_level(logging_level) + + app = _make_app(dask_host, dask_scheduler_port, cache_path) + app.listen(port) + msg=''' +========================================================================= + StochSS-Compute listening on port: %(port)d + Cache directory: %(cache_path)s + Connecting to Dask scheduler at: %(dask_host)s:%(dask_scheduler_port)d +========================================================================= +''' + log.info(msg, locals()) + + try: + await asyncio.Event().wait() + except asyncio.exceptions.CancelledError as error: + log.error(error) + finally: + if rm and os.path.exists(cache_path): + log.info('Removing cache...') + subprocess.Popen(['rm', '-r', cache_path]) + log.info('Cache Removed OK') + \ No newline at end of file diff --git a/gillespy2/remote/server/cache.py b/gillespy2/remote/server/cache.py new file mode 100644 index 00000000..c9cb8177 --- /dev/null +++ b/gillespy2/remote/server/cache.py @@ -0,0 +1,194 @@ +''' +gillespy2.remote.server.cache +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +from json.decoder import JSONDecodeError +import random +from filelock import SoftFileLock +from gillespy2 import Results + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class Cache: + ''' + Cache + + :param cache_dir: The root cache directory. + :type cache_dir: str + + :param results_id: Simulation hash. + :type results_id: str + + :param namespace: Optional namespace. + :type namespace: str + ''' + def __init__(self, cache_dir, results_id, namespace=None): + if namespace is not None: + namespaced_dir = os.path.join(cache_dir, namespace) + cache_dir = namespaced_dir + log.debug(namespaced_dir) + if not os.path.exists(cache_dir): + os.makedirs(cache_dir) + self.results_path = os.path.join(cache_dir, f'{results_id}.results') + + def create(self) -> None: + ''' + Create the results file if it does not exist. + ''' + try: + with open(self.results_path, 'x', encoding='utf-8') as file: + file.close() + except FileExistsError: + pass + + def exists(self) -> bool: + ''' + Check if the results file exists. + + :returns: os.path.exists(self.results_path) + :rtype: bool + ''' + return os.path.exists(self.results_path) + + def is_empty(self) -> bool: + ''' + Check if the results are empty. + + :returns: filesize == 0 or self.exists() + :rtype: bool + ''' + lock = SoftFileLock(f'{self.results_path}.lock') + with lock: + if self.exists(): + filesize = os.path.getsize(self.results_path) + return filesize == 0 + return True + + def is_ready(self, n_traj_wanted=0) -> bool: + ''' + Check if the results are ready to be retrieved from the cache. + + :param n_traj_wanted: The number of requested trajectories. If set to 0, check if any. + :type: int + + :returns: n_traj_wanted <= len() + :rtype: bool + ''' + len_results = self.n_traj_in_cache() + if n_traj_wanted == 0 and len_results > 0: + return True + if n_traj_wanted <= len_results: + return True + return False + + def n_traj_needed(self, n_traj_wanted) -> int: + ''' + Calculate the difference between the number of trajectories the user has requested + and the number of trajectories currently in the cache. + + :param n_traj_wanted: The number of requested trajectories. + :type: int + + :returns: A number greater than or equal to zero. + :rtype: int + ''' + if self.is_empty(): + return n_traj_wanted + results = self.get() + if results is None: + return n_traj_wanted + diff = n_traj_wanted - len(results) + if diff > 0: + return diff + return 0 + + def n_traj_in_cache(self) -> int: + ''' + Check the number of trajectories in the cache. + + :returns: `len()` of the gillespy2.Results + :rtype: int + ''' + if self.is_empty(): + return 0 + results = self.get() + if results is not None: + return len(results) + return 0 + + def get(self): + ''' + Retrieve a gillespy2.Results object from the cache or None if error. + + :returns: Results.from_json(results_json) + :rtype: gillespy2.Results or None + ''' + try: + results_json = self.read() + return Results.from_json(results_json) + except JSONDecodeError: + return None + + def get_sample(self, n_traj) -> Results or None: + ''' + Retrieve a gillespy2.Results by sampling from the cache or None if error. + + :returns: Results.from_json(results_json) + :rtype: gillespy2.Results or None + ''' + results = self.get() + if results is None or len(results) <= n_traj: + return results + ret_traj = random.sample(results, n_traj) + return Results(ret_traj) + + def read(self) -> str: + ''' + Retrieve a gillespy2.Results object as a JSON-formatted string. + + :returns: The output of reading the file. + :rtype: str + ''' + lock = SoftFileLock(f'{self.results_path}.lock') + with lock: + with open(self.results_path,'r', encoding='utf-8') as file: + return file.read() + + def save(self, results: Results) -> None: + ''' + Save a newly processed gillespy2.Results object to the cache. + + :param results: The new Results. + :type: gillespy2.Results + ''' + msg = f'<{self.results_path}> | ' + lock = SoftFileLock(f'{self.results_path}.lock') + with lock: + with open(self.results_path, 'r+', encoding='utf-8') as file: + try: + old_results = Results.from_json(file.read()) + combined_results = results + old_results + log.info(msg+'Add') + file.seek(0) + file.write(combined_results.to_json()) + except JSONDecodeError: + log.info(msg+'New') + file.seek(0) + file.write(results.to_json()) diff --git a/gillespy2/remote/server/handlers/is_cached.py b/gillespy2/remote/server/handlers/is_cached.py new file mode 100644 index 00000000..b8dcb7cb --- /dev/null +++ b/gillespy2/remote/server/handlers/is_cached.py @@ -0,0 +1,87 @@ +''' +gillespy2.remote.server.is_cached +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from datetime import datetime +from tornado.web import RequestHandler +from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.messages.status import SimStatus, StatusResponse +from gillespy2.remote.server.cache import Cache + +class IsCachedHandler(RequestHandler): + ''' + Endpoint that will determine if a particular simulation exists in the cache. + ''' + def initialize(self, cache_dir): + ''' + Set cache path. + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + self.cache_dir = cache_dir + + async def get(self, results_id = None, n_traj = None): + ''' + Process GET request. + + :param results_id: Hash of the simulation. + :type results_id: str + + :param n_traj: Number of trajectories to check for. + :type n_traj: str + ''' + if None in (results_id, n_traj): + raise RemoteSimulationError('Malformed request') + n_traj = int(n_traj) + cache = Cache(self.cache_dir, results_id) + print(f'\ +{datetime.now()} |\ + Source: <{self.request.remote_ip}> |\ + Cache Check |\ + <{results_id}> |\ + Trajectories: {n_traj} ') + msg = f'{datetime.now()} | <{results_id}> | Trajectories: {n_traj} | Status: ' + exists = cache.exists() + if exists: + empty = cache.is_empty() + if empty: + print(msg+SimStatus.DOES_NOT_EXIST.name) + self._respond_dne('That simulation is not currently cached.') + else: + ready = cache.is_ready(n_traj) + if ready: + print(msg+SimStatus.READY.name) + self._respond_ready() + else: + print(msg+SimStatus.DOES_NOT_EXIST.name) + self._respond_dne(f'Not enough trajectories in cache. \ + Requested: {n_traj}, Available: {cache.n_traj_in_cache()}') + else: + print(msg+SimStatus.DOES_NOT_EXIST.name) + self._respond_dne('There is no record of that simulation') + + def _respond_ready(self): + status_response = StatusResponse(SimStatus.READY) + self.write(status_response.encode()) + self.finish() + + def _respond_dne(self, msg): + status_response = StatusResponse(SimStatus.DOES_NOT_EXIST, msg) + self.write(status_response.encode()) + self.finish() diff --git a/gillespy2/remote/server/handlers/number_of_trajectories.py b/gillespy2/remote/server/handlers/number_of_trajectories.py new file mode 100644 index 00000000..be3e3b41 --- /dev/null +++ b/gillespy2/remote/server/handlers/number_of_trajectories.py @@ -0,0 +1,62 @@ +''' +gillespy2.remote.server.number_of_trajectories +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tornado.web import RequestHandler + +from gillespy2.remote.core.utils.log_config import init_logging +from gillespy2.remote.server.cache import Cache +log = init_logging(__name__) + +class NumberOfTrajectoriesHandler(RequestHandler): + ''' + Responds with the number of trajectories in the cache. + ''' + def __init__(self, application, request, **kwargs): + self.cache_dir = None + super().__init__(application, request, **kwargs) + + def data_received(self, chunk: bytes): + raise NotImplementedError() + + def initialize(self, cache_dir): + ''' + Set the cache directory. + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + self.cache_dir = cache_dir + + def get(self): + ''' + Process GET request. + + :returns: Number of trajectories in the cache. + :rtype: str + ''' + msg = f'Request from {self.request.remote_ip}' + log.info(msg) + results_id = self.get_query_argument('results_id') + namespace = self.get_query_argument('namespace', None) + msg = f'results_id: <{results_id}> | namespace: <{namespace}>' + log.info(msg) + cache = Cache(self.cache_dir, results_id, namespace=namespace) + n_traj_in_cache = cache.n_traj_in_cache() + self.write(str(n_traj_in_cache)) + self.finish() diff --git a/gillespy2/remote/server/handlers/number_of_workers.py b/gillespy2/remote/server/handlers/number_of_workers.py new file mode 100644 index 00000000..11d6b8f6 --- /dev/null +++ b/gillespy2/remote/server/handlers/number_of_workers.py @@ -0,0 +1,57 @@ +''' +gillespy2.remote.server.sourceip +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tornado.web import RequestHandler +from distributed import Client + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class NumberOfWorkersHandler(RequestHandler): + ''' + Responds with the number of workers attached to the scheduler. + ''' + scheduler_address = None + def initialize(self, scheduler_address): + ''' + Sets the address to the Dask scheduler and the cache directory. + + :param scheduler_address: Scheduler address. + :type scheduler_address: str + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + + self.scheduler_address = scheduler_address + + def get(self): + ''' + Process POST request. + + :returns: request.remote_ip + :rtype: str + ''' + msg = f' <{self.request.remote_ip}>' + log.info(msg) + client = Client(self.scheduler_address) + n_workers = len(client.scheduler_info().get('workers', [])) + self.write(str(n_workers)) + self.finish() + client.close() diff --git a/gillespy2/remote/server/handlers/results.py b/gillespy2/remote/server/handlers/results.py new file mode 100644 index 00000000..e70dab9a --- /dev/null +++ b/gillespy2/remote/server/handlers/results.py @@ -0,0 +1,81 @@ +''' +gillespy2.remote.server.results +''' +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from tornado.web import RequestHandler +from gillespy2.remote.core.errors import RemoteSimulationError +from gillespy2.remote.core.messages.results import ResultsResponse +from gillespy2.remote.server.cache import Cache + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class ResultsHandler(RequestHandler): + ''' + Endpoint for Results objects. + ''' + def __init__(self, application, request, **kwargs): + self.cache_dir = None + super().__init__(application, request, **kwargs) + + def data_received(self, chunk: bytes): + raise NotImplementedError() + + def initialize(self, cache_dir): + ''' + Set the cache directory. + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + self.cache_dir = cache_dir + + async def get(self): + ''' + Process GET request. + + :param results_id: Hash of the simulation. + :type results_id: str + + :param n_traj: Number of trajectories in the request. + :type n_traj: str + ''' + results_id = self.get_query_argument('results_id', None) + n_traj = self.get_query_argument('n_traj', 0) + n_traj = int(n_traj) + namespace = self.get_query_argument('namespace', None) + if results_id is None: + self.set_status(404, reason=f'Malformed request: {self.request.uri}') + self.finish() + raise RemoteSimulationError(f'Malformed request | <{self.request.remote_ip}>') + msg = f' <{self.request.remote_ip}> | Results Request | <{results_id}>' + log.info(msg) + cache = Cache(self.cache_dir, results_id, namespace=namespace) + if cache.is_ready(n_traj_wanted=n_traj): + log.debug('cache.is_ready()...') + log.debug('n_traj') + log.debug(n_traj) + if n_traj > 0: + results = cache.get_sample(n_traj) + results = results.to_json() + else: + results = cache.read() + results_response = ResultsResponse(results) + self.write(results_response.encode()) + else: + self.set_status(404, f'Results "{results_id}" not found.') + self.finish() diff --git a/gillespy2/remote/server/handlers/simulation_run_cache.py b/gillespy2/remote/server/handlers/simulation_run_cache.py new file mode 100644 index 00000000..a7ce62e4 --- /dev/null +++ b/gillespy2/remote/server/handlers/simulation_run_cache.py @@ -0,0 +1,257 @@ +''' +gillespy2.remote.server.run +''' +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +import random + +from tornado.web import RequestHandler +from tornado.ioloop import IOLoop +from distributed import Client +from gillespy2.remote.core.messages.status import SimStatus +from gillespy2.remote.core.messages.simulation_run_cache import SimulationRunCacheRequest, SimulationRunCacheResponse +from gillespy2.remote.server.cache import Cache + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class SimulationRunCacheHandler(RequestHandler): + ''' + Endpoint for running Gillespy2 simulations as normal, + except that trajectories are cached and reused + if a particular model is run multiple times. + ''' + + scheduler_address = None + cache_dir = None + + def initialize(self, scheduler_address, cache_dir): + ''' + Sets the address to the Dask scheduler and the cache directory. + + :param scheduler_address: Scheduler address. + :type scheduler_address: str + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + + self.scheduler_address = scheduler_address + self.cache_dir = cache_dir + + async def post(self): + ''' + Process simulation run request. + ''' + sim_request = SimulationRunCacheRequest.parse(self.request.body) + log.debug('sim_request...') + log.debug('.results_id:') + log.debug(sim_request.results_id) + log.debug('.id:') + log.debug(sim_request.id) + + results_id = sim_request.results_id + self.cache = Cache(self.cache_dir, results_id, namespace=sim_request.namespace) + msg_0 = f'<{self.request.remote_ip}> | <{results_id}>' + if not self.cache.exists(): + self.cache.create() + empty = self.cache.is_empty() + # Check the number of trajectories in the request, default 1 + n_traj = sim_request.kwargs.get('number_of_trajectories', 1) + if sim_request.force_run is True: + msg = f'{msg_0} | Force run simulation. Running {n_traj} new trajectories.' + log.info(msg) + self._run_cache(sim_request) + elif not empty: + # Compare that to the number of cached trajectories + trajectories_needed = self.cache.n_traj_needed(n_traj) + if trajectories_needed > 0: + sim_request.kwargs['number_of_trajectories'] = trajectories_needed + msg = f'{msg_0} | Partial cache. Running {trajectories_needed} new trajectories.' + log.info(msg) + self._run_cache(sim_request) + else: + msg = f'{msg_0} | Returning cached results.' + log.info(msg) + results = self.cache.get_sample(n_traj) + results_json = results.to_json() + sim_response = SimulationRunCacheResponse(SimStatus.READY, results_id = results_id, results = results_json) + self.write(sim_response.encode()) + self.finish() + elif empty: + msg = f'{msg_0} | Results not cached. Running simulation.' + log.info(msg) + self._run_cache(sim_request) + + def _run_cache(self, sim_request) -> None: + ''' + :param results_id: Key to results. + :type results_id: str + + :param future: Future that completes to gillespy2.Results. + :type future: distributed.Future + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + ''' + results_id = sim_request.results_id + client = Client(self.scheduler_address) + log.debug('_run_cache():') + log.debug(sim_request.parallelize) + log.debug(sim_request.chunk_trajectories) + if sim_request.parallelize is True: + self._submit_parallel(sim_request, client) + elif sim_request.chunk_trajectories is True: + self._submit_chunks(sim_request, client) + else: + future = self._submit(sim_request, client) + self._return_running(results_id, future.key) + IOLoop.current().run_in_executor(None, self._cache, future, client) + + def _submit_parallel(self, sim_request, client: Client): + ''' + :param sim_request: Incoming request. + :type sim_request: SimulationRunCacheRequest + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + :returns: Future that completes to gillespy2.Results. + :rtype: distributed.Future + ''' + model = sim_request.model + kwargs = sim_request.kwargs + results_id = sim_request.results_id + if "solver" in kwargs: + from pydoc import locate + kwargs["solver"] = locate(kwargs["solver"]) + + futures = [] + n_traj = kwargs['number_of_trajectories'] + del kwargs['number_of_trajectories'] + kwargs['seed'] = int(sim_request.id, 16) + for i in range(n_traj): + kwargs['seed'] += kwargs['seed'] + future = client.submit(model.run, **kwargs) + futures.append(future) + IOLoop.current().run_in_executor(None, self._cache_parallel, futures, client) + self._return_running(results_id, results_id) + + def _submit_chunks(self, sim_request, client: Client): + ''' + :param sim_request: Incoming request. + :type sim_request: SimulationRunCacheRequest + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + :returns: Future that completes to gillespy2.Results. + :rtype: distributed.Future + ''' + model = sim_request.model + kwargs = sim_request.kwargs + results_id = sim_request.results_id + if "solver" in kwargs: + from pydoc import locate + kwargs["solver"] = locate(kwargs["solver"]) + n_traj = kwargs.get('number_of_trajectories',1) + n_workers = len(client.scheduler_info()['workers']) + traj_per_worker = n_traj // n_workers + extra_traj = n_traj % n_workers + log.debug('_submit_chunks():') + log.debug(traj_per_worker) + log.debug(extra_traj) + futures = [] + del kwargs['number_of_trajectories'] + kwargs['seed'] = int(sim_request.id, 16) + for _ in range(n_workers): + kwargs['seed'] += kwargs['seed'] + kwargs['number_of_trajectories'] = traj_per_worker + future = client.submit(model.run, **kwargs) + futures.append(future) + if extra_traj > 0: + kwargs['seed'] += kwargs['seed'] + kwargs['number_of_trajectories'] = extra_traj + future = client.submit(model.run, **kwargs) + futures.append(future) + + IOLoop.current().run_in_executor(None, self._cache_parallel, futures, client) + self._return_running(results_id, results_id) + + + + def _cache_parallel(self, futures, client: Client) -> None: + ''' + :param results_id: Key to results. + :type results_id: str + + :param future: List of Futures that completes to gillespy2.Results. + :type : List[distributed.Future] + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + ''' + log.debug('_cache_parallel()...') + results = client.gather(futures, asynchronous=False) + results = sum(results) + log.debug('results:') + log.debug(results) + client.close() + self.cache.save(results) + + def _cache(self, future, client) -> None: + ''' + :param future: Future that completes to gillespy2.Results. + :type future: distributed.Future + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + ''' + results = future.result() + client.close() + self.cache.save(results) + + def _submit(self, sim_request, client: Client): + ''' + :param sim_request: Incoming request. + :type sim_request: SimulationRunCacheRequest + + :param client: Handle to dask scheduler. + :type client: distributed.Client + + :returns: Future that completes to gillespy2.Results. + :rtype: distributed.Future + ''' + log.debug('_submit()...') + model = sim_request.model + kwargs = sim_request.kwargs + + if "solver" in kwargs: + from pydoc import locate + kwargs["solver"] = locate(kwargs["solver"]) + log.debug('kwargs.get("solver", None)') + log.debug(kwargs.get('solver', None)) + + return client.submit(model.run, **kwargs, key=sim_request.id) + + def _return_running(self, results_id, task_id): + sim_response = SimulationRunCacheResponse(SimStatus.RUNNING, results_id=results_id, task_id=task_id) + self.write(sim_response.encode()) + self.finish() diff --git a/gillespy2/remote/server/handlers/sourceip.py b/gillespy2/remote/server/handlers/sourceip.py new file mode 100644 index 00000000..d63c6a72 --- /dev/null +++ b/gillespy2/remote/server/handlers/sourceip.py @@ -0,0 +1,53 @@ +''' +gillespy2.remote.server.sourceip +''' +# StochSS-Compute is a tool for running and caching GillesPy2 simulations remotely. +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import os +from tornado.web import RequestHandler +from gillespy2.remote.core.messages.source_ip import SourceIpRequest, SourceIpResponse + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class SourceIpHandler(RequestHandler): + ''' + Responds with the IP address associated with the request. + Used only by cloud api. + ''' + + def post(self): + ''' + Process POST request. + + :returns: request.remote_ip + :rtype: str + ''' + source_ip = self.request.remote_ip + msg = f'Request from {source_ip}' + log.info(msg) + source_ip_request = SourceIpRequest.parse(self.request.body) + if source_ip_request.cloud_key == os.environ.get('CLOUD_LOCK'): + msg = f'{source_ip} Access granted.' + log.info(msg) + source_ip_response = SourceIpResponse(source_ip=source_ip) + self.write(source_ip_response.encode()) + else: + msg = f'{source_ip} Access denied.' + log.info(msg) + self.set_status(403, 'Access denied.') + self.finish() diff --git a/gillespy2/remote/server/handlers/status.py b/gillespy2/remote/server/handlers/status.py new file mode 100644 index 00000000..4c9f5039 --- /dev/null +++ b/gillespy2/remote/server/handlers/status.py @@ -0,0 +1,163 @@ +''' +gillespy2.remote.server.status +''' +# Copyright (C) 2019-2023 GillesPy2 and StochSS developers. + +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. + +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. + +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from distributed import Client +from tornado.web import RequestHandler +from gillespy2.remote.core.messages.status import SimStatus, StatusResponse +from gillespy2.remote.server.cache import Cache + +from gillespy2.remote.core.utils.log_config import init_logging +log = init_logging(__name__) + +class StatusHandler(RequestHandler): + ''' + Endpoint for requesting the status of a simulation. + ''' + def __init__(self, application, request, **kwargs): + self.scheduler_address = None + self.cache_dir = None + super().__init__(application, request, **kwargs) + + def data_received(self, chunk: bytes): + raise NotImplementedError() + + def initialize(self, scheduler_address, cache_dir): + ''' + Sets the address to the Dask scheduler and the cache directory. + + :param scheduler_address: Scheduler address. + :type scheduler_address: str + + :param cache_dir: Path to the cache. + :type cache_dir: str + ''' + self.scheduler_address = scheduler_address + self.cache_dir = cache_dir + + async def get(self): + ''' + Process Status GET request. + ''' + results_id = self.get_query_argument('results_id') + n_traj = self.get_query_argument('n_traj', 0) + if n_traj is not None: + n_traj = int(n_traj) + task_id = self.get_query_argument('task_id', None) + namespace = self.get_query_argument('namespace', None) + + cache = Cache(self.cache_dir, results_id, namespace=namespace) + + msg_0 = f'<{self.request.remote_ip}> | Results ID: <{results_id}> | Trajectories: {n_traj} | Task ID: {task_id}' + log.info(msg_0) + + msg_1 = f'<{results_id}> | <{task_id}> | Status:' + running_msg = f'{msg_1} {SimStatus.RUNNING.name}' + dne_msg = f'{msg_1} {SimStatus.DOES_NOT_EXIST.name}' + ready_msg = f'{msg_1} {SimStatus.READY.name}' + + if cache.exists(): + log.debug('cache.exists(): True') + + if cache.is_empty(): + + if task_id is not None: + + state, err = await self._check_with_scheduler(task_id) + + msg_2 = f'{msg_1} {SimStatus.RUNNING.name} | Task: {state} | Error: {err}' + log.info(msg_2) + + if state == 'erred': + self._respond_error(err) + else: + self._respond_running(f'Scheduler Task State: {state}') + + else: + + log.info(running_msg) + self._respond_running('Simulation still running.') + + else: + + if cache.is_ready(n_traj): + log.info(ready_msg) + self._respond_ready() + + else: + + if task_id is not None: + + state, err = await self._check_with_scheduler(task_id) + + msg_2 = f'{msg_1} {SimStatus.RUNNING.name} | Task: {state} | Error: {err}' + log.info(msg_2) + + if state == 'erred': + self._respond_error(err) + else: + self._respond_running(f'Scheduler task state: {state}') + + else: + + log.info(running_msg) + self._respond_running('Simulation still running.') + + else: + log.debug('cache.exists(): False') + log.info(dne_msg) + self._respond_dne() + + def _respond_ready(self): + status_response = StatusResponse(SimStatus.READY) + self.write(status_response.encode()) + self.finish() + + def _respond_error(self, error_message): + status_response = StatusResponse(SimStatus.ERROR, error_message) + self.write(status_response.encode()) + self.finish() + + def _respond_dne(self): + status_response = StatusResponse(SimStatus.DOES_NOT_EXIST, 'There is no record of that simulation.') + self.write(status_response.encode()) + self.finish() + + def _respond_running(self, message): + status_response = StatusResponse(SimStatus.RUNNING, message) + self.write(status_response.encode()) + self.finish() + + async def _check_with_scheduler(self, task_id): + ''' + Ask the scheduler for information about a task. + ''' + client = Client(self.scheduler_address) + + # define function here so that it is pickle-able + def scheduler_task_state(task_id, dask_scheduler=None): + task = dask_scheduler.tasks.get(task_id) + if task is None: + return (None, None) + if task.exception_text == "": + return (task.state, None) + return (task.state, task.exception_text) + + # Do not await. Reasons. It returns sync. + _ = client.run_on_scheduler(scheduler_task_state, task_id) + client.close() + return _ diff --git a/mdip.demo.ipynb b/mdip.demo.ipynb new file mode 100644 index 00000000..6b163899 --- /dev/null +++ b/mdip.demo.ipynb @@ -0,0 +1,270 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-07-19 10:21:30,998 - gillespy2.remote.cloud.ec2 - WARNING - boto3 and parkamiko are required for gillespy2.remote.cloud.ec2\n" + ] + } + ], + "source": [ + "import gillespy2\n", + "import gillespy2.remote as remote\n", + "import gillespy2.remote.core.utils.debug " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def create_michaelis_menten(parameter_values=None):\n", + " # Intialize the Model with a name of your choosing.\n", + " model = gillespy2.Model(name=\"Michaelis_Menten\")\n", + "\n", + " \"\"\"\n", + " Variables (GillesPy2.Species) can be anything that participates in or is produced by a reaction channel.\n", + "\n", + " - name: A user defined name for the species.\n", + " - initial_value: A value/population count of species at start of simulation.\n", + " \"\"\"\n", + " A = gillespy2.Species(name=\"A\", initial_value=301)\n", + " B = gillespy2.Species(name=\"B\", initial_value=120)\n", + " C = gillespy2.Species(name=\"C\", initial_value=0)\n", + " D = gillespy2.Species(name=\"D\", initial_value=0)\n", + "\n", + " # Add the Variables to the Model.\n", + " model.add_species([A, B, C, D])\n", + "\n", + " \"\"\"\n", + " Parameters are constant values relevant to the system, such as reaction kinetic rates.\n", + "\n", + " - name: A user defined name for reference.\n", + " - expression: Some constant value.\n", + " \"\"\"\n", + " rate1 = gillespy2.Parameter(name=\"rate1\", expression=0.0017)\n", + " rate2 = gillespy2.Parameter(name=\"rate2\", expression=0.5)\n", + " rate3 = gillespy2.Parameter(name=\"rate3\", expression=0.1)\n", + "\n", + " # Add the Parameters to the Model.\n", + " model.add_parameter([rate1, rate2, rate3])\n", + "\n", + " \"\"\"\n", + " Reactions are the reaction channels which cause the system to change over time.\n", + "\n", + " - name: A user defined name for the reaction.\n", + " - reactants: A dictionary with participant reactants as keys, and consumed per reaction as value.\n", + " - products: A dictionary with reaction products as keys, and number formed per reaction as value.\n", + " - rate: A parameter rate constant to be applied to the propensity of this reaction firing.\n", + " - propensity_function: Can be used instead of rate in order to declare a custom propensity function in string format.\n", + " \"\"\"\n", + " r1 = gillespy2.Reaction(\n", + " name=\"r1\",\n", + " reactants={'A': 1, 'B': 1}, \n", + " products={'C': 1},\n", + " rate='rate1'\n", + " # propensity_function='1/0' # uncomment to cause error.\n", + " )\n", + "\n", + " r2 = gillespy2.Reaction(\n", + " name=\"r2\",\n", + " reactants={'C': 1}, \n", + " products={'A': 1, 'B': 1},\n", + " rate='rate2'\n", + " )\n", + "\n", + " r3 = gillespy2.Reaction(\n", + " name=\"r3\",\n", + " reactants={'C': 1}, \n", + " products={'B': 1, 'D': 1},\n", + " rate='rate3'\n", + " )\n", + "\n", + " # Add the Reactions to the Model.\n", + " model.add_reaction([r1, r2, r3])\n", + "\n", + " # Define the timespan of the model.\n", + " tspan = gillespy2.TimeSpan.linspace(t=100, num_points=100)\n", + " \n", + " # Set the timespan of the Model.\n", + " model.timespan(tspan)\n", + " return model" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "scrolled": true, + "tags": [] + }, + "outputs": [], + "source": [ + "myModel = create_michaelis_menten()" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "simulation = remote.RemoteSimulation(myModel, host='localhost', solver=gillespy2.NumPySSASolver)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-07-19 10:21:43,205 - gillespy2.remote.client.server - DEBUG - {'model': '{\\n \"_generate_translation_table\": true,\\n \"_hash_private_vars\": false,\\n \"_listOfAssignmentRules\": {},\\n \"_listOfEvents\": {},\\n \"_listOfFunctionDefinitions\": {},\\n \"_listOfParameters\": {\\n \"rate1\": \"P0\",\\n \"rate2\": \"P1\",\\n \"rate3\": \"P2\"\\n },\\n \"_listOfRateRules\": {},\\n \"_listOfReactions\": {\\n \"r1\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": null,\\n \"massaction\": false,\\n \"name\": \"R1\",\\n \"ode_propensity_function\": \"(((P0*S0)*S1)/vol)\",\\n \"products\": [\\n {\\n \"key\": \"S2\",\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(((P0*S0)*S1)/vol)\",\\n \"reactants\": [\\n {\\n \"key\": \"S0\",\\n \"value\": 1\\n },\\n {\\n \"key\": \"S1\",\\n \"value\": 1\\n }\\n ],\\n \"type\": \"customized\"\\n },\\n \"r2\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": null,\\n \"massaction\": false,\\n \"name\": \"R2\",\\n \"ode_propensity_function\": \"(P1*S2)\",\\n \"products\": [\\n {\\n \"key\": \"S0\",\\n \"value\": 1\\n },\\n {\\n \"key\": \"S1\",\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(P1*S2)\",\\n \"reactants\": [\\n {\\n \"key\": \"S2\",\\n \"value\": 1\\n }\\n ],\\n \"type\": \"customized\"\\n },\\n \"r3\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": null,\\n \"massaction\": false,\\n \"name\": \"R3\",\\n \"ode_propensity_function\": \"(P2*S2)\",\\n \"products\": [\\n {\\n \"key\": \"S1\",\\n \"value\": 1\\n },\\n {\\n \"key\": \"S3\",\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(P2*S2)\",\\n \"reactants\": [\\n {\\n \"key\": \"S2\",\\n \"value\": 1\\n }\\n ],\\n \"type\": \"customized\"\\n }\\n },\\n \"_listOfSpecies\": {\\n \"A\": \"S0\",\\n \"B\": \"S1\",\\n \"C\": \"S2\",\\n \"D\": \"S3\"\\n },\\n \"_type\": \"gillespy2.core.model.Model\",\\n \"annotation\": \"model\",\\n \"listOfAssignmentRules\": {},\\n \"listOfEvents\": {},\\n \"listOfFunctionDefinitions\": {},\\n \"listOfParameters\": {\\n \"rate1\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.0017\",\\n \"name\": \"rate1\",\\n \"value\": 0.0017\\n },\\n \"rate2\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.5\",\\n \"name\": \"rate2\",\\n \"value\": 0.5\\n },\\n \"rate3\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.1\",\\n \"name\": \"rate3\",\\n \"value\": 0.1\\n }\\n },\\n \"listOfRateRules\": {},\\n \"listOfReactions\": {\\n \"r1\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.0017\",\\n \"name\": \"rate1\",\\n \"value\": 0.0017\\n },\\n \"massaction\": true,\\n \"name\": \"r1\",\\n \"ode_propensity_function\": \"((rate1*A)*B)\",\\n \"products\": [\\n {\\n \"key\": {\\n \"_hash\": -5406967187597232989,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"C\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(((rate1*A)*B)/vol)\",\\n \"reactants\": [\\n {\\n \"key\": {\\n \"_hash\": 1689242874924442423,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 301,\\n \"mode\": null,\\n \"name\": \"A\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n },\\n {\\n \"key\": {\\n \"_hash\": 1522382677053280009,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 120,\\n \"mode\": null,\\n \"name\": \"B\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"type\": \"mass-action\"\\n },\\n \"r2\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.5\",\\n \"name\": \"rate2\",\\n \"value\": 0.5\\n },\\n \"massaction\": true,\\n \"name\": \"r2\",\\n \"ode_propensity_function\": \"(rate2*C)\",\\n \"products\": [\\n {\\n \"key\": {\\n \"_hash\": 1689242874924442423,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 301,\\n \"mode\": null,\\n \"name\": \"A\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n },\\n {\\n \"key\": {\\n \"_hash\": 1522382677053280009,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 120,\\n \"mode\": null,\\n \"name\": \"B\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(rate2*C)\",\\n \"reactants\": [\\n {\\n \"key\": {\\n \"_hash\": -5406967187597232989,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"C\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"type\": \"mass-action\"\\n },\\n \"r3\": {\\n \"_type\": \"gillespy2.core.reaction.Reaction\",\\n \"annotation\": null,\\n \"marate\": {\\n \"_type\": \"gillespy2.core.parameter.Parameter\",\\n \"expression\": \"0.1\",\\n \"name\": \"rate3\",\\n \"value\": 0.1\\n },\\n \"massaction\": true,\\n \"name\": \"r3\",\\n \"ode_propensity_function\": \"(rate3*C)\",\\n \"products\": [\\n {\\n \"key\": {\\n \"_hash\": 1522382677053280009,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 120,\\n \"mode\": null,\\n \"name\": \"B\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n },\\n {\\n \"key\": {\\n \"_hash\": 8606284047498760059,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"D\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"propensity_function\": \"(rate3*C)\",\\n \"reactants\": [\\n {\\n \"key\": {\\n \"_hash\": -5406967187597232989,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"C\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"value\": 1\\n }\\n ],\\n \"type\": \"mass-action\"\\n }\\n },\\n \"listOfSpecies\": {\\n \"A\": {\\n \"_hash\": 1689242874924442423,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 301,\\n \"mode\": null,\\n \"name\": \"A\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"B\": {\\n \"_hash\": 1522382677053280009,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 120,\\n \"mode\": null,\\n \"name\": \"B\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"C\": {\\n \"_hash\": -5406967187597232989,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"C\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n },\\n \"D\": {\\n \"_hash\": 8606284047498760059,\\n \"_type\": \"gillespy2.core.species.Species\",\\n \"allow_negative_populations\": false,\\n \"boundary_condition\": false,\\n \"constant\": false,\\n \"initial_value\": 0,\\n \"mode\": null,\\n \"name\": \"D\",\\n \"switch_min\": 0,\\n \"switch_tol\": 0.03\\n }\\n },\\n \"name\": \"Michaelis_Menten\",\\n \"namespace\": {\\n \"rate1\": 0.0017,\\n \"rate2\": 0.5\\n },\\n \"tspan\": {\\n \"_type\": \"gillespy2.core.timespan.TimeSpan\",\\n \"items\": {\\n \"_type\": \"gillespy2.core.jsonify.NdArrayCoder\",\\n \"data\": [\\n 0.0,\\n 1.0101010101010102,\\n 2.0202020202020203,\\n 3.0303030303030303,\\n 4.040404040404041,\\n 5.050505050505051,\\n 6.0606060606060606,\\n 7.070707070707071,\\n 8.080808080808081,\\n 9.090909090909092,\\n 10.101010101010102,\\n 11.111111111111112,\\n 12.121212121212121,\\n 13.131313131313131,\\n 14.141414141414142,\\n 15.151515151515152,\\n 16.161616161616163,\\n 17.171717171717173,\\n 18.181818181818183,\\n 19.191919191919194,\\n 20.202020202020204,\\n 21.212121212121215,\\n 22.222222222222225,\\n 23.232323232323235,\\n 24.242424242424242,\\n 25.252525252525253,\\n 26.262626262626263,\\n 27.272727272727273,\\n 28.282828282828284,\\n 29.292929292929294,\\n 30.303030303030305,\\n 31.313131313131315,\\n 32.323232323232325,\\n 33.333333333333336,\\n 34.343434343434346,\\n 35.35353535353536,\\n 36.36363636363637,\\n 37.37373737373738,\\n 38.38383838383839,\\n 39.3939393939394,\\n 40.40404040404041,\\n 41.41414141414142,\\n 42.42424242424243,\\n 43.43434343434344,\\n 44.44444444444445,\\n 45.45454545454546,\\n 46.46464646464647,\\n 47.47474747474748,\\n 48.484848484848484,\\n 49.494949494949495,\\n 50.505050505050505,\\n 51.515151515151516,\\n 52.525252525252526,\\n 53.535353535353536,\\n 54.54545454545455,\\n 55.55555555555556,\\n 56.56565656565657,\\n 57.57575757575758,\\n 58.58585858585859,\\n 59.5959595959596,\\n 60.60606060606061,\\n 61.61616161616162,\\n 62.62626262626263,\\n 63.63636363636364,\\n 64.64646464646465,\\n 65.65656565656566,\\n 66.66666666666667,\\n 67.67676767676768,\\n 68.68686868686869,\\n 69.6969696969697,\\n 70.70707070707071,\\n 71.71717171717172,\\n 72.72727272727273,\\n 73.73737373737374,\\n 74.74747474747475,\\n 75.75757575757576,\\n 76.76767676767678,\\n 77.77777777777779,\\n 78.7878787878788,\\n 79.7979797979798,\\n 80.80808080808082,\\n 81.81818181818183,\\n 82.82828282828284,\\n 83.83838383838385,\\n 84.84848484848486,\\n 85.85858585858587,\\n 86.86868686868688,\\n 87.87878787878789,\\n 88.8888888888889,\\n 89.89898989898991,\\n 90.90909090909092,\\n 91.91919191919193,\\n 92.92929292929294,\\n 93.93939393939395,\\n 94.94949494949496,\\n 95.95959595959597,\\n 96.96969696969697,\\n 97.97979797979798,\\n 98.98989898989899,\\n 100.0\\n ]\\n }\\n },\\n \"units\": \"population\",\\n \"volume\": 1.0\\n}', 'kwargs': {'solver': 'gillespy2.solvers.numpy.ssa_solver.NumPySSASolver'}, 'id': 'a25f451380dc2ebe2049918d33ac6c67', 'results_id': 'a25f451380dc2ebe2049918d33ac6c67', 'namespace': None, 'ignore_cache': True}\n", + "2023-07-19 10:21:43,206 - gillespy2.remote.client.server - DEBUG - http://localhost:29681/api/v3/simulation/gillespy2/run/cache\n", + "2023-07-19 10:21:43,211 - urllib3.connectionpool - DEBUG - Starting new HTTP connection (1): localhost:29681\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[SimulationRunCacheRequest] http://localhost:29681/api/v3/simulation/gillespy2/run/cache\n", + "Unknown error: HTTPConnectionPool(host='localhost', port=29681): Max retries exceeded with url: /api/v3/simulation/gillespy2/run/cache (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')). Retrying in 3 seconds....\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-07-19 10:21:46,226 - urllib3.connectionpool - DEBUG - Starting new HTTP connection (1): localhost:29681\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[SimulationRunCacheRequest] http://localhost:29681/api/v3/simulation/gillespy2/run/cache\n", + "Unknown error: HTTPConnectionPool(host='localhost', port=29681): Max retries exceeded with url: /api/v3/simulation/gillespy2/run/cache (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')). Retrying in 6 seconds....\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2023-07-19 10:21:52,232 - urllib3.connectionpool - DEBUG - Starting new HTTP connection (1): localhost:29681\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[SimulationRunCacheRequest] http://localhost:29681/api/v3/simulation/gillespy2/run/cache\n", + "Unknown error: HTTPConnectionPool(host='localhost', port=29681): Max retries exceeded with url: /api/v3/simulation/gillespy2/run/cache (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused')). Retrying in 18 seconds....\n" + ] + }, + { + "ename": "KeyboardInterrupt", + "evalue": "", + "output_type": "error", + "traceback": [ + "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", + "\u001b[0;31mConnectionRefusedError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connection.py:174\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 173\u001b[0m \u001b[39mtry\u001b[39;00m:\n\u001b[0;32m--> 174\u001b[0m conn \u001b[39m=\u001b[39m connection\u001b[39m.\u001b[39;49mcreate_connection(\n\u001b[1;32m 175\u001b[0m (\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_dns_host, \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mport), \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mtimeout, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mextra_kw\n\u001b[1;32m 176\u001b[0m )\n\u001b[1;32m 178\u001b[0m \u001b[39mexcept\u001b[39;00m SocketTimeout:\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/util/connection.py:95\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m 94\u001b[0m \u001b[39mif\u001b[39;00m err \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[0;32m---> 95\u001b[0m \u001b[39mraise\u001b[39;00m err\n\u001b[1;32m 97\u001b[0m \u001b[39mraise\u001b[39;00m socket\u001b[39m.\u001b[39merror(\u001b[39m\"\u001b[39m\u001b[39mgetaddrinfo returns an empty list\u001b[39m\u001b[39m\"\u001b[39m)\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/util/connection.py:85\u001b[0m, in \u001b[0;36mcreate_connection\u001b[0;34m(address, timeout, source_address, socket_options)\u001b[0m\n\u001b[1;32m 84\u001b[0m sock\u001b[39m.\u001b[39mbind(source_address)\n\u001b[0;32m---> 85\u001b[0m sock\u001b[39m.\u001b[39;49mconnect(sa)\n\u001b[1;32m 86\u001b[0m \u001b[39mreturn\u001b[39;00m sock\n", + "\u001b[0;31mConnectionRefusedError\u001b[0m: [Errno 111] Connection refused", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mNewConnectionError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connectionpool.py:714\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m 713\u001b[0m \u001b[39m# Make the request on the httplib connection object.\u001b[39;00m\n\u001b[0;32m--> 714\u001b[0m httplib_response \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_make_request(\n\u001b[1;32m 715\u001b[0m conn,\n\u001b[1;32m 716\u001b[0m method,\n\u001b[1;32m 717\u001b[0m url,\n\u001b[1;32m 718\u001b[0m timeout\u001b[39m=\u001b[39;49mtimeout_obj,\n\u001b[1;32m 719\u001b[0m body\u001b[39m=\u001b[39;49mbody,\n\u001b[1;32m 720\u001b[0m headers\u001b[39m=\u001b[39;49mheaders,\n\u001b[1;32m 721\u001b[0m chunked\u001b[39m=\u001b[39;49mchunked,\n\u001b[1;32m 722\u001b[0m )\n\u001b[1;32m 724\u001b[0m \u001b[39m# If we're going to release the connection in ``finally:``, then\u001b[39;00m\n\u001b[1;32m 725\u001b[0m \u001b[39m# the response doesn't need to know about the connection. Otherwise\u001b[39;00m\n\u001b[1;32m 726\u001b[0m \u001b[39m# it will also try to release it and we'll have a double-release\u001b[39;00m\n\u001b[1;32m 727\u001b[0m \u001b[39m# mess.\u001b[39;00m\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connectionpool.py:415\u001b[0m, in \u001b[0;36mHTTPConnectionPool._make_request\u001b[0;34m(self, conn, method, url, timeout, chunked, **httplib_request_kw)\u001b[0m\n\u001b[1;32m 414\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[0;32m--> 415\u001b[0m conn\u001b[39m.\u001b[39;49mrequest(method, url, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mhttplib_request_kw)\n\u001b[1;32m 417\u001b[0m \u001b[39m# We are swallowing BrokenPipeError (errno.EPIPE) since the server is\u001b[39;00m\n\u001b[1;32m 418\u001b[0m \u001b[39m# legitimately able to close the connection after sending a valid response.\u001b[39;00m\n\u001b[1;32m 419\u001b[0m \u001b[39m# With this behaviour, the received response is still readable.\u001b[39;00m\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connection.py:244\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers)\u001b[0m\n\u001b[1;32m 243\u001b[0m headers[\u001b[39m\"\u001b[39m\u001b[39mUser-Agent\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m _get_default_user_agent()\n\u001b[0;32m--> 244\u001b[0m \u001b[39msuper\u001b[39;49m(HTTPConnection, \u001b[39mself\u001b[39;49m)\u001b[39m.\u001b[39;49mrequest(method, url, body\u001b[39m=\u001b[39;49mbody, headers\u001b[39m=\u001b[39;49mheaders)\n", + "File \u001b[0;32m~/.python/cpython.dip.build.3.11/lib/python3.11/http/client.py:1282\u001b[0m, in \u001b[0;36mHTTPConnection.request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m 1281\u001b[0m \u001b[39m\u001b[39m\u001b[39m\"\"\"Send a complete request to the server.\"\"\"\u001b[39;00m\n\u001b[0;32m-> 1282\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_request(method, url, body, headers, encode_chunked)\n", + "File \u001b[0;32m~/.python/cpython.dip.build.3.11/lib/python3.11/http/client.py:1328\u001b[0m, in \u001b[0;36mHTTPConnection._send_request\u001b[0;34m(self, method, url, body, headers, encode_chunked)\u001b[0m\n\u001b[1;32m 1327\u001b[0m body \u001b[39m=\u001b[39m _encode(body, \u001b[39m'\u001b[39m\u001b[39mbody\u001b[39m\u001b[39m'\u001b[39m)\n\u001b[0;32m-> 1328\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mendheaders(body, encode_chunked\u001b[39m=\u001b[39;49mencode_chunked)\n", + "File \u001b[0;32m~/.python/cpython.dip.build.3.11/lib/python3.11/http/client.py:1277\u001b[0m, in \u001b[0;36mHTTPConnection.endheaders\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1276\u001b[0m \u001b[39mraise\u001b[39;00m CannotSendHeader()\n\u001b[0;32m-> 1277\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_send_output(message_body, encode_chunked\u001b[39m=\u001b[39;49mencode_chunked)\n", + "File \u001b[0;32m~/.python/cpython.dip.build.3.11/lib/python3.11/http/client.py:1037\u001b[0m, in \u001b[0;36mHTTPConnection._send_output\u001b[0;34m(self, message_body, encode_chunked)\u001b[0m\n\u001b[1;32m 1036\u001b[0m \u001b[39mdel\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_buffer[:]\n\u001b[0;32m-> 1037\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(msg)\n\u001b[1;32m 1039\u001b[0m \u001b[39mif\u001b[39;00m message_body \u001b[39mis\u001b[39;00m \u001b[39mnot\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m 1040\u001b[0m \n\u001b[1;32m 1041\u001b[0m \u001b[39m# create a consistent interface to message_body\u001b[39;00m\n", + "File \u001b[0;32m~/.python/cpython.dip.build.3.11/lib/python3.11/http/client.py:975\u001b[0m, in \u001b[0;36mHTTPConnection.send\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m 974\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mauto_open:\n\u001b[0;32m--> 975\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mconnect()\n\u001b[1;32m 976\u001b[0m \u001b[39melse\u001b[39;00m:\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connection.py:205\u001b[0m, in \u001b[0;36mHTTPConnection.connect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 204\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mconnect\u001b[39m(\u001b[39mself\u001b[39m):\n\u001b[0;32m--> 205\u001b[0m conn \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_new_conn()\n\u001b[1;32m 206\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_prepare_conn(conn)\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connection.py:186\u001b[0m, in \u001b[0;36mHTTPConnection._new_conn\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 185\u001b[0m \u001b[39mexcept\u001b[39;00m SocketError \u001b[39mas\u001b[39;00m e:\n\u001b[0;32m--> 186\u001b[0m \u001b[39mraise\u001b[39;00m NewConnectionError(\n\u001b[1;32m 187\u001b[0m \u001b[39mself\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mFailed to establish a new connection: \u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m\"\u001b[39m \u001b[39m%\u001b[39m e\n\u001b[1;32m 188\u001b[0m )\n\u001b[1;32m 190\u001b[0m \u001b[39mreturn\u001b[39;00m conn\n", + "\u001b[0;31mNewConnectionError\u001b[0m: : Failed to establish a new connection: [Errno 111] Connection refused", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mMaxRetryError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/adapters.py:489\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 488\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m chunked:\n\u001b[0;32m--> 489\u001b[0m resp \u001b[39m=\u001b[39m conn\u001b[39m.\u001b[39;49murlopen(\n\u001b[1;32m 490\u001b[0m method\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mmethod,\n\u001b[1;32m 491\u001b[0m url\u001b[39m=\u001b[39;49murl,\n\u001b[1;32m 492\u001b[0m body\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mbody,\n\u001b[1;32m 493\u001b[0m headers\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mheaders,\n\u001b[1;32m 494\u001b[0m redirect\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 495\u001b[0m assert_same_host\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 496\u001b[0m preload_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 497\u001b[0m decode_content\u001b[39m=\u001b[39;49m\u001b[39mFalse\u001b[39;49;00m,\n\u001b[1;32m 498\u001b[0m retries\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mmax_retries,\n\u001b[1;32m 499\u001b[0m timeout\u001b[39m=\u001b[39;49mtimeout,\n\u001b[1;32m 500\u001b[0m )\n\u001b[1;32m 502\u001b[0m \u001b[39m# Send the request.\u001b[39;00m\n\u001b[1;32m 503\u001b[0m \u001b[39melse\u001b[39;00m:\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/connectionpool.py:798\u001b[0m, in \u001b[0;36mHTTPConnectionPool.urlopen\u001b[0;34m(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)\u001b[0m\n\u001b[1;32m 796\u001b[0m e \u001b[39m=\u001b[39m ProtocolError(\u001b[39m\"\u001b[39m\u001b[39mConnection aborted.\u001b[39m\u001b[39m\"\u001b[39m, e)\n\u001b[0;32m--> 798\u001b[0m retries \u001b[39m=\u001b[39m retries\u001b[39m.\u001b[39;49mincrement(\n\u001b[1;32m 799\u001b[0m method, url, error\u001b[39m=\u001b[39;49me, _pool\u001b[39m=\u001b[39;49m\u001b[39mself\u001b[39;49m, _stacktrace\u001b[39m=\u001b[39;49msys\u001b[39m.\u001b[39;49mexc_info()[\u001b[39m2\u001b[39;49m]\n\u001b[1;32m 800\u001b[0m )\n\u001b[1;32m 801\u001b[0m retries\u001b[39m.\u001b[39msleep()\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/urllib3/util/retry.py:592\u001b[0m, in \u001b[0;36mRetry.increment\u001b[0;34m(self, method, url, response, error, _pool, _stacktrace)\u001b[0m\n\u001b[1;32m 591\u001b[0m \u001b[39mif\u001b[39;00m new_retry\u001b[39m.\u001b[39mis_exhausted():\n\u001b[0;32m--> 592\u001b[0m \u001b[39mraise\u001b[39;00m MaxRetryError(_pool, url, error \u001b[39mor\u001b[39;00m ResponseError(cause))\n\u001b[1;32m 594\u001b[0m log\u001b[39m.\u001b[39mdebug(\u001b[39m\"\u001b[39m\u001b[39mIncremented Retry for (url=\u001b[39m\u001b[39m'\u001b[39m\u001b[39m%s\u001b[39;00m\u001b[39m'\u001b[39m\u001b[39m): \u001b[39m\u001b[39m%r\u001b[39;00m\u001b[39m\"\u001b[39m, url, new_retry)\n", + "\u001b[0;31mMaxRetryError\u001b[0m: HTTPConnectionPool(host='localhost', port=29681): Max retries exceeded with url: /api/v3/simulation/gillespy2/run/cache (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mConnectionError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/GillesPy2/gillespy2/remote/client/server.py:121\u001b[0m, in \u001b[0;36mServer.post\u001b[0;34m(self, endpoint, sub, request)\u001b[0m\n\u001b[1;32m 120\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m[\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mtype\u001b[39m(request)\u001b[39m.\u001b[39m\u001b[39m__name__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m] \u001b[39m\u001b[39m{\u001b[39;00murl\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 121\u001b[0m \u001b[39mreturn\u001b[39;00m requests\u001b[39m.\u001b[39;49mpost(url, json\u001b[39m=\u001b[39;49mrequest\u001b[39m.\u001b[39;49mencode(), timeout\u001b[39m=\u001b[39;49m\u001b[39m30\u001b[39;49m)\n\u001b[1;32m 123\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mConnectionError\u001b[39;00m:\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/api.py:115\u001b[0m, in \u001b[0;36mpost\u001b[0;34m(url, data, json, **kwargs)\u001b[0m\n\u001b[1;32m 104\u001b[0m \u001b[39m\u001b[39m\u001b[39mr\u001b[39m\u001b[39m\"\"\"Sends a POST request.\u001b[39;00m\n\u001b[1;32m 105\u001b[0m \n\u001b[1;32m 106\u001b[0m \u001b[39m:param url: URL for the new :class:`Request` object.\u001b[39;00m\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 112\u001b[0m \u001b[39m:rtype: requests.Response\u001b[39;00m\n\u001b[1;32m 113\u001b[0m \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m--> 115\u001b[0m \u001b[39mreturn\u001b[39;00m request(\u001b[39m\"\u001b[39;49m\u001b[39mpost\u001b[39;49m\u001b[39m\"\u001b[39;49m, url, data\u001b[39m=\u001b[39;49mdata, json\u001b[39m=\u001b[39;49mjson, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/api.py:59\u001b[0m, in \u001b[0;36mrequest\u001b[0;34m(method, url, **kwargs)\u001b[0m\n\u001b[1;32m 58\u001b[0m \u001b[39mwith\u001b[39;00m sessions\u001b[39m.\u001b[39mSession() \u001b[39mas\u001b[39;00m session:\n\u001b[0;32m---> 59\u001b[0m \u001b[39mreturn\u001b[39;00m session\u001b[39m.\u001b[39;49mrequest(method\u001b[39m=\u001b[39;49mmethod, url\u001b[39m=\u001b[39;49murl, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/sessions.py:587\u001b[0m, in \u001b[0;36mSession.request\u001b[0;34m(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)\u001b[0m\n\u001b[1;32m 586\u001b[0m send_kwargs\u001b[39m.\u001b[39mupdate(settings)\n\u001b[0;32m--> 587\u001b[0m resp \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msend(prep, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49msend_kwargs)\n\u001b[1;32m 589\u001b[0m \u001b[39mreturn\u001b[39;00m resp\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/sessions.py:701\u001b[0m, in \u001b[0;36mSession.send\u001b[0;34m(self, request, **kwargs)\u001b[0m\n\u001b[1;32m 700\u001b[0m \u001b[39m# Send the request\u001b[39;00m\n\u001b[0;32m--> 701\u001b[0m r \u001b[39m=\u001b[39m adapter\u001b[39m.\u001b[39;49msend(request, \u001b[39m*\u001b[39;49m\u001b[39m*\u001b[39;49mkwargs)\n\u001b[1;32m 703\u001b[0m \u001b[39m# Total elapsed time of the request (approximately)\u001b[39;00m\n", + "File \u001b[0;32m~/GillesPy2/env/lib/python3.11/site-packages/requests/adapters.py:565\u001b[0m, in \u001b[0;36mHTTPAdapter.send\u001b[0;34m(self, request, stream, timeout, verify, cert, proxies)\u001b[0m\n\u001b[1;32m 563\u001b[0m \u001b[39mraise\u001b[39;00m SSLError(e, request\u001b[39m=\u001b[39mrequest)\n\u001b[0;32m--> 565\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mConnectionError\u001b[39;00m(e, request\u001b[39m=\u001b[39mrequest)\n\u001b[1;32m 567\u001b[0m \u001b[39mexcept\u001b[39;00m ClosedPoolError \u001b[39mas\u001b[39;00m e:\n", + "\u001b[0;31mConnectionError\u001b[0m: HTTPConnectionPool(host='localhost', port=29681): Max retries exceeded with url: /api/v3/simulation/gillespy2/run/cache (Caused by NewConnectionError(': Failed to establish a new connection: [Errno 111] Connection refused'))", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn[6], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m results \u001b[39m=\u001b[39m simulation\u001b[39m.\u001b[39;49mrun(ignore_cache\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n", + "File \u001b[0;32m~/GillesPy2/gillespy2/remote/core/remote_simulation.py:134\u001b[0m, in \u001b[0;36mRemoteSimulation.run\u001b[0;34m(self, namespace, ignore_cache, **params)\u001b[0m\n\u001b[1;32m 132\u001b[0m params[\u001b[39m\"\u001b[39m\u001b[39msolver\u001b[39m\u001b[39m\"\u001b[39m] \u001b[39m=\u001b[39m \u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msolver\u001b[39m.\u001b[39m\u001b[39m__module__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m.\u001b[39m\u001b[39m{\u001b[39;00m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39msolver\u001b[39m.\u001b[39m\u001b[39m__qualname__\u001b[39m\u001b[39m}\u001b[39;00m\u001b[39m\"\u001b[39m\n\u001b[1;32m 133\u001b[0m sim_request \u001b[39m=\u001b[39m SimulationRunCacheRequest(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mmodel, namespace\u001b[39m=\u001b[39mnamespace, ignore_cache\u001b[39m=\u001b[39mignore_cache, \u001b[39m*\u001b[39m\u001b[39m*\u001b[39mparams)\n\u001b[0;32m--> 134\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_run_cache(sim_request)\n", + "File \u001b[0;32m~/GillesPy2/gillespy2/remote/core/remote_simulation.py:141\u001b[0m, in \u001b[0;36mRemoteSimulation._run_cache\u001b[0;34m(self, request)\u001b[0m\n\u001b[1;32m 136\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_run_cache\u001b[39m(\u001b[39mself\u001b[39m, request):\n\u001b[1;32m 137\u001b[0m \u001b[39m \u001b[39m\u001b[39m'''\u001b[39;00m\n\u001b[1;32m 138\u001b[0m \u001b[39m :param request: Request to send to the server. Contains Model and related arguments.\u001b[39;00m\n\u001b[1;32m 139\u001b[0m \u001b[39m :type request: SimulationRunRequest\u001b[39;00m\n\u001b[1;32m 140\u001b[0m \u001b[39m '''\u001b[39;00m\n\u001b[0;32m--> 141\u001b[0m response_raw \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49mserver\u001b[39m.\u001b[39;49mpost(Endpoint\u001b[39m.\u001b[39;49mSIMULATION_GILLESPY2, sub\u001b[39m=\u001b[39;49m\u001b[39m'\u001b[39;49m\u001b[39m/run/cache\u001b[39;49m\u001b[39m'\u001b[39;49m, request\u001b[39m=\u001b[39;49mrequest)\n\u001b[1;32m 143\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m response_raw\u001b[39m.\u001b[39mok:\n\u001b[1;32m 144\u001b[0m \u001b[39mraise\u001b[39;00m \u001b[39mException\u001b[39;00m(response_raw\u001b[39m.\u001b[39mreason)\n", + "File \u001b[0;32m~/GillesPy2/gillespy2/remote/client/server.py:130\u001b[0m, in \u001b[0;36mServer.post\u001b[0;34m(self, endpoint, sub, request)\u001b[0m\n\u001b[1;32m 128\u001b[0m \u001b[39mexcept\u001b[39;00m \u001b[39mException\u001b[39;00m \u001b[39mas\u001b[39;00m err:\n\u001b[1;32m 129\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m\u001b[39mUnknown error: \u001b[39m\u001b[39m{\u001b[39;00merr\u001b[39m}\u001b[39;00m\u001b[39m. Retrying in \u001b[39m\u001b[39m{\u001b[39;00msec\u001b[39m}\u001b[39;00m\u001b[39m seconds....\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m--> 130\u001b[0m sleep(sec)\n\u001b[1;32m 131\u001b[0m n_try \u001b[39m+\u001b[39m\u001b[39m=\u001b[39m \u001b[39m1\u001b[39m\n\u001b[1;32m 132\u001b[0m sec \u001b[39m*\u001b[39m\u001b[39m=\u001b[39m n_try\n", + "\u001b[0;31mKeyboardInterrupt\u001b[0m: " + ] + } + ], + "source": [ + "results = simulation.run(ignore_cache=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "results.plot()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2+" + }, + "vscode": { + "interpreter": { + "hash": "ef56f9d787682a6ac61c29852cd545f557bd05dd1aa39793e08a6cd6ebb9b699" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/setup.py b/setup.py index 7ea5c7e8..c6ba3672 100644 --- a/setup.py +++ b/setup.py @@ -85,6 +85,13 @@ "Source" : "https://github.com/StochSS/GillesPy2", }, packages = find_packages('.'), + entry_points={ + 'console_scripts': [ + 'gillespy2-remote=gillespy2.remote.launch:launch_server', + 'gillespy2-remote-cluster=gillespy2.remote.launch:launch_with_cluster', + ] + }, + include_package_data = True, install_requires = reqs, @@ -99,10 +106,25 @@ 'Topic :: Scientific/Engineering :: Medical Science Apps.', 'Intended Audience :: Science/Research' ], - extras_requires = { + extras_require = { 'sbml': [ 'python_libsbml', 'lxml', ], + 'remote': [ + 'distributed == 2022.12.1', + 'requests == 2.28.1', + 'filelock == 3.9.0' + ], + 'aws': [ + 'boto3 == 1.24.71', + 'paramiko == 2.11.0', + 'python-dotenv == 0.21.0' + ], + 'remote.dev': [ + 'coverage', + 'moto == 4.1.0', + ] + }, )