From d975d06148b715e8aa1b4ed4ae3d0ac4cda1a461 Mon Sep 17 00:00:00 2001 From: dweinholz Date: Thu, 25 Jan 2024 09:20:32 +0100 Subject: [PATCH] refactor(Config):more comments, bibigrid and forc not required --- .../bibigrid_connector/bibigrid_connector.py | 13 ++- simple_vm_client/config/config.yml | 93 +++++++++++-------- simple_vm_client/config/config_local.yml | 72 +++++++------- .../forc_connector/forc_connector.py | 18 +++- .../forc_connector/template/template.py | 35 +++---- .../forc_connector/template/test_templates.py | 6 +- .../forc_connector/test_forc_connector.py | 4 +- .../openstack_connector.py | 8 +- 8 files changed, 151 insertions(+), 98 deletions(-) diff --git a/simple_vm_client/bibigrid_connector/bibigrid_connector.py b/simple_vm_client/bibigrid_connector/bibigrid_connector.py index 547ad33..3f05319 100644 --- a/simple_vm_client/bibigrid_connector/bibigrid_connector.py +++ b/simple_vm_client/bibigrid_connector/bibigrid_connector.py @@ -24,8 +24,16 @@ def __init__(self, config_file: str): def load_config_yml(self, config_file: str) -> None: with open(config_file, "r") as ymlfile: cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader) + # Check if "bibigrid" key is present in the loaded YAML + if "bibigrid" not in cfg: + # Optionally, you can log a message or take other actions here + logger.info("Bibigrid configuration not found. Skipping.") + return bibigrid_cfg = cfg["bibigrid"] + if not bibigrid_cfg.get("activated", True): + logger.info("Bibigrid Config available but deactivated. Skipping..") + return self._BIBIGRID_HOST = bibigrid_cfg["host"] self._BIBIGRID_PORT = bibigrid_cfg["port"] self._BIBIGRID_USE_HTTPS = bibigrid_cfg.get("https", False) @@ -33,12 +41,13 @@ def load_config_yml(self, config_file: str) -> None: self._BIBIGRID_USE_MASTER_WITH_PUBLIC_IP = bibigrid_cfg.get( "use_master_with_public_ip", False ) + self._SUB_NETWORK = bibigrid_cfg["sub_network"] + self._BIBIGRID_LOCAL_DNS_LOOKUP = bibigrid_cfg.get("localDnsLookup", False) self._BIBIGRID_ANSIBLE_ROLES = bibigrid_cfg.get("ansibleGalaxyRoles", []) openstack_cfg = cfg["openstack"] self._NETWORK = openstack_cfg["network"] - self._SUB_NETWORK = openstack_cfg["sub_network"] self._PRODUCTION = cfg["production"] protocol = "https" if self._BIBIGRID_USE_HTTPS else "http" @@ -116,7 +125,7 @@ def get_clusters_info(self) -> list[dict[str, str]]: return infos def is_bibigrid_available(self) -> bool: - logger.info("Checking if Bibigrid is available") + logger.info(f"Checking if Bibigrid is available via: {self._BIBIGRID_EP}") if not self._BIBIGRID_EP: logger.info("Bibigrid Url is not set") diff --git a/simple_vm_client/config/config.yml b/simple_vm_client/config/config.yml index 1ad559f..829fd55 100644 --- a/simple_vm_client/config/config.yml +++ b/simple_vm_client/config/config.yml @@ -1,63 +1,82 @@ +# Configuration for production environment production: False +# Set to True if this configuration is intended for a production environment, otherwise set to False for development/testing. - - +# Redis configuration redis: host: simplevm_client_redis + # Hostname or IP address of the Redis server. port: 6379 + # Port number on which the Redis server is running. password: "" - - + # Password for authenticating to the Redis server. Leave empty if no password is required. + +# Logger configuration +logger: + level: INFO + # Logging level, e.g., INFO, DEBUG, WARNING, ERROR. + file: log/portal_client.log + # Path to the log file. + file_backup_count: 5 + # Number of backup log files to keep. + max_bytes: 1073741824 + # Maximum size (in bytes) of each log file before it's rotated. + +# Server configuration server: threads: 30 + # Number of threads to use for the server. host: 0.0.0.0 + # Host IP address to bind the server to. port: 9090 - # If you use docker-compose this path needs to be the path you mount the server.pem into + # Port number on which the server will listen. certfile: /code/VirtualMachineService/keys/server.pem - use_ssl: True - - + # Path to the SSL certificate file. Used if use_ssl is set to True. + use_ssl: False + # Set to True if SSL should be used, otherwise False. +# OpenStack configuration openstack: - gateway_security_group_id: 88b530b3-0b62-4edc-b5f6-d80fbbba0830 + gateway_security_group_id: 88b530b3-0b62-4edc-b5f6-d80fbbba0830 + # Security group ID for the gateway. + forc_security_group_id: 9a08eecc-d9a5-405b-aeda-9d4180fc94d6 + # Security group ID for FORC cloud_site: bielefeld - #Calculation for Ports - 30000 Base Port (192.168.1.20 - 1 = y , 20 =x) + # Name or identifier of the OpenStack cloud site. ssh_port_calculation: 30000 + x + y * 256 + # Calculation for determining the SSH port. Schema -> 192.168.y.x udp_port_calculation: 30000 + x + y * 256 - - - # Gateway IP + # Calculation for determining the UDP port. Schema -> 192.168.y.x gateway_ip: 129.70.51.75 - # If set to True the client will use a Gateway instead of providing floating IPs for each instance. - use_gateway: True - - set_password: False - # network where the project is located + # IP address of the OpenStack gateway. network: portalexternalnetwork + # Name or identifier of the openstack network - # subnetwork for starting Clusters +# Bibigrid configuration +bibigrid: + activated: False + # If Bibigrid ist activated -- defaults to True + port: 8080 + # Port number for Bibigrid API. + host: simplevm_bibigrid + # Hostname or IP address of the Bibigrid server. + https: False + # Set to True if Bibigrid should use HTTPS, otherwise False. + localDnsLookup: False + # Set to True if Bibigrid should perform local DNS lookup, otherwise False. sub_network: portalexternalsubnetwork + # Name or identifier of the openstack subnetwork. + modes: + - slurm - production: False - - - floating_ip_network: external - - # If you use docker-compose this path needs to be the path you mount the server.pem into - certfile: /code/VirtualMachineService/keys/server.pem - use_ssl: True - -bibigrid: - # Url for Bibigrid API - port: 8080 - host: simplevm_bibigrid - https: False - sub_network: portalexternalsubnetwork - modes: - - slurm +# FORC forc: + activated: True + # If Forc ist activated -- defaults to True forc_url: https://proxy-dev.bi.denbi.de:5000/ + # URL for FORC. forc_access_url: https://proxy-dev.bi.denbi.de/ + # Access URL for FORC - optional - default is forc_url without port. github_playbooks_repo: https://github.com/deNBI/resenvs/archive/refs/heads/staging.zip - forc_security_group_id: forc_security_group_id + # GitHub repository URL for FORC playbooks. diff --git a/simple_vm_client/config/config_local.yml b/simple_vm_client/config/config_local.yml index 3043b0c..829fd55 100644 --- a/simple_vm_client/config/config_local.yml +++ b/simple_vm_client/config/config_local.yml @@ -1,72 +1,82 @@ +# Configuration for production environment production: False +# Set to True if this configuration is intended for a production environment, otherwise set to False for development/testing. - +# Redis configuration redis: host: simplevm_client_redis + # Hostname or IP address of the Redis server. port: 6379 + # Port number on which the Redis server is running. password: "" + # Password for authenticating to the Redis server. Leave empty if no password is required. +# Logger configuration logger: level: INFO + # Logging level, e.g., INFO, DEBUG, WARNING, ERROR. file: log/portal_client.log + # Path to the log file. file_backup_count: 5 + # Number of backup log files to keep. max_bytes: 1073741824 + # Maximum size (in bytes) of each log file before it's rotated. +# Server configuration server: threads: 30 + # Number of threads to use for the server. host: 0.0.0.0 + # Host IP address to bind the server to. port: 9090 - # If you use docker-compose this path needs to be the path you mount the server.pem into + # Port number on which the server will listen. certfile: /code/VirtualMachineService/keys/server.pem + # Path to the SSL certificate file. Used if use_ssl is set to True. use_ssl: False + # Set to True if SSL should be used, otherwise False. +# OpenStack configuration openstack: - gateway_security_group_id: 88b530b3-0b62-4edc-b5f6-d80fbbba0830 + # Security group ID for the gateway. + forc_security_group_id: 9a08eecc-d9a5-405b-aeda-9d4180fc94d6 + # Security group ID for FORC cloud_site: bielefeld - #Calculation for Ports - 30000 Base Port (192.168.1.20 - 1 = y , 20 =x) + # Name or identifier of the OpenStack cloud site. ssh_port_calculation: 30000 + x + y * 256 + # Calculation for determining the SSH port. Schema -> 192.168.y.x udp_port_calculation: 30000 + x + y * 256 - - - # Gateway IP + # Calculation for determining the UDP port. Schema -> 192.168.y.x gateway_ip: 129.70.51.75 - # If set to True the client will use a Gateway instead of providing floating IPs for each instance. - use_gateway: True - - set_password: False - # network where the project is located + # IP address of the OpenStack gateway. network: portalexternalnetwork + # Name or identifier of the openstack network - # subnetwork for starting Clusters - sub_network: portalexternalsubnetwork - - - floating_ip_network: external - - - # Used for gateway port calculation , x is always the last octet of the fixed ip (example : (x + 3) *3 ) ) - ssh_port_calc_formular: 30000 + x - udp_port_calc_formular: x * 10 + 30000 - +# Bibigrid configuration bibigrid: - # Url for Bibigrid API + activated: False + # If Bibigrid ist activated -- defaults to True port: 8080 + # Port number for Bibigrid API. host: simplevm_bibigrid + # Hostname or IP address of the Bibigrid server. https: False + # Set to True if Bibigrid should use HTTPS, otherwise False. localDnsLookup: False + # Set to True if Bibigrid should perform local DNS lookup, otherwise False. sub_network: portalexternalsubnetwork + # Name or identifier of the openstack subnetwork. modes: - slurm - ansibleGalaxyRoles: - - name: autoscaling - hosts: master - git: https://github.com/patricS4/autoscaling-config-ansible + +# FORC forc: + activated: True + # If Forc ist activated -- defaults to True forc_url: https://proxy-dev.bi.denbi.de:5000/ + # URL for FORC. forc_access_url: https://proxy-dev.bi.denbi.de/ + # Access URL for FORC - optional - default is forc_url without port. github_playbooks_repo: https://github.com/deNBI/resenvs/archive/refs/heads/staging.zip - forc_security_group_id: 9a08eecc-d9a5-405b-aeda-9d4180fc94d6 - -cloud_site: bielefeld + # GitHub repository URL for FORC playbooks. diff --git a/simple_vm_client/forc_connector/forc_connector.py b/simple_vm_client/forc_connector/forc_connector.py index 962a5e0..bf85710 100644 --- a/simple_vm_client/forc_connector/forc_connector.py +++ b/simple_vm_client/forc_connector/forc_connector.py @@ -33,15 +33,14 @@ def __init__(self, config_file: str): self.FORC_URL: str = "" # type: ignore self.FORC_ACCESS_URL: str = "" # type: ignore - self.FORC_REMOTE_ID: str = "" # type: ignore self.GITHUB_PLAYBOOKS_REPO: str = "" # type: ignore self.REDIS_HOST: str = "" # type: ignore self.REDIS_PORT: int = None # type: ignore + self.FORC_API_KEY: str = "" self.redis_pool: redis.ConnectionPool = None # type: ignore self.redis_connection: redis.Redis.connection_pool = None self._active_playbooks: dict[str, Playbook] = {} self.load_config(config_file=config_file) - self.load_env() self.connect_to_redis() self.template = Template( github_playbook_repo=self.GITHUB_PLAYBOOKS_REPO, @@ -53,15 +52,24 @@ def load_config(self, config_file: str) -> None: logger.info("Load config file: FORC") with open(config_file, "r") as ymlfile: cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader) + # Check if "bibigrid" key is present in the loaded YAML + self.REDIS_HOST = cfg["redis"]["host"] + self.REDIS_PORT = cfg["redis"]["port"] + if "forc" not in cfg: + # Optionally, you can log a message or take other actions here + logger.info("Forc configuration not found. Skipping.") + return + if not cfg["forc"].get("activated", True): + logger.info("Forc Config available but deactivated. Skipping..") + return self.FORC_URL = cfg["forc"]["forc_url"] url_components = urlparse(cfg["forc"]["forc_url"]) path = url_components.netloc.split(":")[0] self.FORC_ACCESS_URL = f"{url_components.scheme}://{path}/" - self.FORC_REMOTE_ID = cfg["forc"]["forc_security_group_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] - self.REDIS_HOST = cfg["redis"]["host"] - self.REDIS_PORT = cfg["redis"]["port"] + + self.load_env() def connect_to_redis(self) -> None: logger.info("Connect to redis") diff --git a/simple_vm_client/forc_connector/template/template.py b/simple_vm_client/forc_connector/template/template.py index 2fb191f..9bbf8c2 100644 --- a/simple_vm_client/forc_connector/template/template.py +++ b/simple_vm_client/forc_connector/template/template.py @@ -84,8 +84,10 @@ def __init__(self, github_playbook_repo: str, forc_url: str, forc_api_key: str): self.GITHUB_PLAYBOOKS_REPO = github_playbook_repo self.FORC_URL = forc_url self.FORC_API_KEY = forc_api_key - self.TEMPLATES_URL = f"{self.FORC_URL}templates" - self.BACKENDS_URL = f"{self.FORC_URL}backends" + if not self.FORC_URL: + logger.info("No FORC URL defined. Skipping Forc...") + self.TEMPLATES_URL = f"{self.FORC_URL}templates" if self.FORC_URL else "" + self.BACKENDS_URL = f"{self.FORC_URL}backends" if self.FORC_URL else "" self.BACKENDS_BY_OWNER_URL = f"{self.BACKENDS_URL}/byOwner" self.BACKENDS_BY_TEMPLATE_URL = f"{self.BACKENDS_URL}/byTemplate" self._forc_allowed: dict[str, list[str]] = {} @@ -158,8 +160,8 @@ def _process_template_metadata( ] = template_metadata def update_playbooks(self) -> None: - if self.GITHUB_PLAYBOOKS_REPO is None: - logger.error( + if not self.GITHUB_PLAYBOOKS_REPO: + logger.warning( "Github playbooks repo URL is None. Aborting download of playbooks." ) return @@ -179,18 +181,19 @@ def update_playbooks(self) -> None: logger.info(f"Allowed Forc {self._forc_allowed}") def _get_forc_templates(self) -> list[dict]: - try: - response = requests.get( - self.TEMPLATES_URL, - timeout=(30, 30), - headers={"X-API-KEY": self.FORC_API_KEY}, - verify=True, - ) - response.raise_for_status() # Raise HTTPError for bad responses - return response.json() - except requests.RequestException as e: - logger.exception(f"Error while fetching FORC templates: {e}") - return [] + if self.TEMPLATES_URL: + try: + response = requests.get( + self.TEMPLATES_URL, + timeout=(30, 30), + headers={"X-API-KEY": self.FORC_API_KEY}, + verify=True, + ) + response.raise_for_status() # Raise HTTPError for bad responses + return response.json() + except requests.RequestException as e: + logger.exception(f"Error while fetching FORC templates: {e}") + return [] def cross_check_forc_image(self, tags: list[str]) -> bool: try: diff --git a/simple_vm_client/forc_connector/template/test_templates.py b/simple_vm_client/forc_connector/template/test_templates.py index 1fc2dca..419aa59 100644 --- a/simple_vm_client/forc_connector/template/test_templates.py +++ b/simple_vm_client/forc_connector/template/test_templates.py @@ -266,11 +266,11 @@ def test_update_loaded_templates(self, mock_isdir, mock_listdir): expected_templates = ["template1", "template2"] self.assertEqual(template._all_templates, expected_templates) - @patch("simple_vm_client.forc_connector.template.template.logger.error") - def test_update_playbooks_no_github_repo(self, mock_logger_error): + @patch("simple_vm_client.forc_connector.template.template.logger.warning") + def test_update_playbooks_no_github_repo(self, mock_logger_warning): template = self.init_template() template.update_playbooks() - mock_logger_error.assert_called_once_with( + mock_logger_warning.assert_called_once_with( "Github playbooks repo URL is None. Aborting download of playbooks." ) diff --git a/simple_vm_client/forc_connector/test_forc_connector.py b/simple_vm_client/forc_connector/test_forc_connector.py index e0ea849..7497ae9 100644 --- a/simple_vm_client/forc_connector/test_forc_connector.py +++ b/simple_vm_client/forc_connector/test_forc_connector.py @@ -81,7 +81,9 @@ def test_load_config(self): os.remove(temp_file.name) self.assertEqual(self.forc_connector.FORC_URL, FORC_URL) self.assertEqual(self.forc_connector.FORC_ACCESS_URL, FORC_ACCESS_URL) - self.assertEqual(self.forc_connector.FORC_REMOTE_ID, FORC_SECRUITY_GROUP_ID) + self.assertEqual( + self.forc_connector.FORC_SECURITY_GROUP_ID, FORC_SECRUITY_GROUP_ID + ) self.assertEqual(self.forc_connector.GITHUB_PLAYBOOKS_REPO, GITHUB_REPO) self.assertEqual(self.forc_connector.REDIS_HOST, REDIS_HOST) self.assertEqual(self.forc_connector.REDIS_PORT, REDIS_PORT) diff --git a/simple_vm_client/openstack_connector/openstack_connector.py b/simple_vm_client/openstack_connector/openstack_connector.py index 13fd9d9..1157ad7 100644 --- a/simple_vm_client/openstack_connector/openstack_connector.py +++ b/simple_vm_client/openstack_connector/openstack_connector.py @@ -57,7 +57,6 @@ def __init__(self, config_file: str): logger.info("Initializing OpenStack Connector") self.GATEWAY_IP: str = "" self.NETWORK: str = "" - self.SUB_NETWORK: str = "" self.PRODUCTION: bool = True self.CLOUD_SITE: str = "" self.SSH_MULTIPLICATION_PORT: int = 1 @@ -122,12 +121,15 @@ def load_config_yml(self, config_file: str) -> None: self.GATEWAY_IP = cfg["openstack"]["gateway_ip"] self.NETWORK = cfg["openstack"]["network"] - self.SUB_NETWORK = cfg["openstack"]["sub_network"] self.PRODUCTION = cfg["production"] self.CLOUD_SITE = cfg["openstack"]["cloud_site"] self.SSH_PORT_CALCULATION = cfg["openstack"]["ssh_port_calculation"] self.UDP_PORT_CALCULATION = cfg["openstack"]["udp_port_calculation"] - self.FORC_SECURITY_GROUP_ID = cfg["forc"]["forc_security_group_id"] + self.FORC_SECURITY_GROUP_ID = cfg["openstack"].get( + "forc_security_group_id", None + ) + if not self.FORC_SECURITY_GROUP_ID: + logger.info("No Forc Security Group defined") self.DEFAULT_SECURITY_GROUP_NAME = "defaultSimpleVM" self.DEFAULT_SECURITY_GROUPS = [self.DEFAULT_SECURITY_GROUP_NAME] self.GATEWAY_SECURITY_GROUP_ID = cfg["openstack"][