refactor(Config):more comments, bibigrid and forc not required

simple_vm_client/bibigrid_connector/bibigrid_connector.py

@@ -24,21 +24,30 @@ def __init__(self, config_file: str):
     def load_config_yml(self, config_file: str) -> None:
         with open(config_file, "r") as ymlfile:
             cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader)
+            # Check if "bibigrid" key is present in the loaded YAML
+            if "bibigrid" not in cfg:
+                # Optionally, you can log a message or take other actions here
+                logger.info("Bibigrid configuration not found. Skipping.")
+                return
 
             bibigrid_cfg = cfg["bibigrid"]
+            if not bibigrid_cfg.get("activated", True):
+                logger.info("Bibigrid Config available but deactivated. Skipping..")
+                return
             self._BIBIGRID_HOST = bibigrid_cfg["host"]
             self._BIBIGRID_PORT = bibigrid_cfg["port"]
             self._BIBIGRID_USE_HTTPS = bibigrid_cfg.get("https", False)
             self._BIBIGRID_MODES = bibigrid_cfg["modes"]
             self._BIBIGRID_USE_MASTER_WITH_PUBLIC_IP = bibigrid_cfg.get(
                 "use_master_with_public_ip", False
             )
+            self._SUB_NETWORK = bibigrid_cfg["sub_network"]
+
             self._BIBIGRID_LOCAL_DNS_LOOKUP = bibigrid_cfg.get("localDnsLookup", False)
             self._BIBIGRID_ANSIBLE_ROLES = bibigrid_cfg.get("ansibleGalaxyRoles", [])
 
             openstack_cfg = cfg["openstack"]
             self._NETWORK = openstack_cfg["network"]
-            self._SUB_NETWORK = openstack_cfg["sub_network"]
             self._PRODUCTION = cfg["production"]
 
             protocol = "https" if self._BIBIGRID_USE_HTTPS else "http"
@@ -116,7 +125,7 @@ def get_clusters_info(self) -> list[dict[str, str]]:
         return infos
 
     def is_bibigrid_available(self) -> bool:
-        logger.info("Checking if Bibigrid is available")
+        logger.info(f"Checking if Bibigrid is available via:  {self._BIBIGRID_EP}")
 
         if not self._BIBIGRID_EP:
             logger.info("Bibigrid Url is not set")

simple_vm_client/config/config.yml

@@ -1,63 +1,82 @@
+# Configuration for production environment
 production: False
+# Set to True if this configuration is intended for a production environment, otherwise set to False for development/testing.
 
-
-
+# Redis configuration
 redis:
   host: simplevm_client_redis
+  # Hostname or IP address of the Redis server.
   port: 6379
+  # Port number on which the Redis server is running.
   password: ""
-
-
+  # Password for authenticating to the Redis server. Leave empty if no password is required.
+
+# Logger configuration
+logger:
+  level: INFO
+  # Logging level, e.g., INFO, DEBUG, WARNING, ERROR.
+  file: log/portal_client.log
+  # Path to the log file.
+  file_backup_count: 5
+  # Number of backup log files to keep.
+  max_bytes: 1073741824
+  # Maximum size (in bytes) of each log file before it's rotated.
+
+# Server configuration
 server:
   threads: 30
+  # Number of threads to use for the server.
   host: 0.0.0.0
+  # Host IP address to bind the server to.
   port: 9090
-  # If you use docker-compose this path needs to be the path you mount the server.pem into
+  # Port number on which the server will listen.
   certfile: /code/VirtualMachineService/keys/server.pem
-  use_ssl: True
-
-
+  # Path to the SSL certificate file. Used if use_ssl is set to True.
+  use_ssl: False
+  # Set to True if SSL should be used, otherwise False.
 
+# OpenStack configuration
 openstack:
-  gateway_security_group_id:  88b530b3-0b62-4edc-b5f6-d80fbbba0830
+  gateway_security_group_id: 88b530b3-0b62-4edc-b5f6-d80fbbba0830
+  # Security group ID for the gateway.
+  forc_security_group_id: 9a08eecc-d9a5-405b-aeda-9d4180fc94d6
+  # Security group ID for FORC
   cloud_site: bielefeld
-  #Calculation for Ports - 30000 Base Port (192.168.1.20 - 1 = y , 20 =x)
+  # Name or identifier of the OpenStack cloud site.
   ssh_port_calculation: 30000 + x + y * 256
+  # Calculation for determining the SSH port. Schema -> 192.168.y.x
   udp_port_calculation: 30000 + x + y * 256
-
-
-  # Gateway IP
+  # Calculation for determining the UDP port. Schema -> 192.168.y.x
   gateway_ip: 129.70.51.75
-  # If set to True the client will use a Gateway instead of providing floating IPs for each instance.
-  use_gateway: True
-
-  set_password: False
-  # network where the project is located
+  # IP address of the OpenStack gateway.
   network: portalexternalnetwork
+  # Name or identifier of the openstack network
 
-  # subnetwork for starting Clusters
+# Bibigrid configuration
+bibigrid:
+  activated: False
+  # If Bibigrid ist activated  -- defaults to True
+  port: 8080
+  # Port number for Bibigrid API.
+  host: simplevm_bibigrid
+  # Hostname or IP address of the Bibigrid server.
+  https: False
+  # Set to True if Bibigrid should use HTTPS, otherwise False.
+  localDnsLookup: False
+  # Set to True if Bibigrid should perform local DNS lookup, otherwise False.
   sub_network: portalexternalsubnetwork
+  # Name or identifier of the openstack subnetwork.
+  modes:
+    - slurm
 
-  production: False
-
-
-  floating_ip_network: external
-
-  # If you use docker-compose this path needs to be the path you mount the server.pem into
-  certfile: /code/VirtualMachineService/keys/server.pem
-  use_ssl: True
-
-bibigrid:
-  # Url for Bibigrid API
-    port: 8080
-    host: simplevm_bibigrid
-    https: False
-    sub_network: portalexternalsubnetwork
-    modes:
-      -   slurm
 
+# FORC
 forc:
+  activated: True
+  # If Forc ist activated  -- defaults to True
   forc_url: https://proxy-dev.bi.denbi.de:5000/
+  # URL for FORC.
   forc_access_url: https://proxy-dev.bi.denbi.de/
+  # Access URL for FORC - optional - default is forc_url without port.
   github_playbooks_repo: https://github.com/deNBI/resenvs/archive/refs/heads/staging.zip
-  forc_security_group_id: forc_security_group_id
+  # GitHub repository URL for FORC playbooks.

simple_vm_client/config/config_local.yml

@@ -1,72 +1,82 @@
+# Configuration for production environment
 production: False
+# Set to True if this configuration is intended for a production environment, otherwise set to False for development/testing.
 
-
+# Redis configuration
 redis:
   host: simplevm_client_redis
+  # Hostname or IP address of the Redis server.
   port: 6379
+  # Port number on which the Redis server is running.
   password: ""
+  # Password for authenticating to the Redis server. Leave empty if no password is required.
 
+# Logger configuration
 logger:
   level: INFO
+  # Logging level, e.g., INFO, DEBUG, WARNING, ERROR.
   file: log/portal_client.log
+  # Path to the log file.
   file_backup_count: 5
+  # Number of backup log files to keep.
   max_bytes: 1073741824
+  # Maximum size (in bytes) of each log file before it's rotated.
 
+# Server configuration
 server:
   threads: 30
+  # Number of threads to use for the server.
   host: 0.0.0.0
+  # Host IP address to bind the server to.
   port: 9090
-  # If you use docker-compose this path needs to be the path you mount the server.pem into
+  # Port number on which the server will listen.
   certfile: /code/VirtualMachineService/keys/server.pem
+  # Path to the SSL certificate file. Used if use_ssl is set to True.
   use_ssl: False
+  # Set to True if SSL should be used, otherwise False.
 
+# OpenStack configuration
 openstack:
-
   gateway_security_group_id: 88b530b3-0b62-4edc-b5f6-d80fbbba0830
+  # Security group ID for the gateway.
+  forc_security_group_id: 9a08eecc-d9a5-405b-aeda-9d4180fc94d6
+  # Security group ID for FORC
   cloud_site: bielefeld
-  #Calculation for Ports - 30000 Base Port (192.168.1.20 - 1 = y , 20 =x)
+  # Name or identifier of the OpenStack cloud site.
   ssh_port_calculation: 30000 + x + y * 256
+  # Calculation for determining the SSH port. Schema -> 192.168.y.x
   udp_port_calculation: 30000 + x + y * 256
-
-
-  # Gateway IP
+  # Calculation for determining the UDP port. Schema -> 192.168.y.x
   gateway_ip: 129.70.51.75
-  # If set to True the client will use a Gateway instead of providing floating IPs for each instance.
-  use_gateway: True
-
-  set_password: False
-  # network where the project is located
+  # IP address of the OpenStack gateway.
   network: portalexternalnetwork
+  # Name or identifier of the openstack network
 
-  # subnetwork for starting Clusters
-  sub_network: portalexternalsubnetwork
-
-
-  floating_ip_network: external
-
-
-  # Used for gateway port calculation , x is always the last octet of the fixed ip (example : (x + 3) *3 ) )
-  ssh_port_calc_formular: 30000 + x
-  udp_port_calc_formular: x * 10 +  30000
-
+# Bibigrid configuration
 bibigrid:
-  # Url for Bibigrid API
+  activated: False
+  # If Bibigrid ist activated  -- defaults to True
   port: 8080
+  # Port number for Bibigrid API.
   host: simplevm_bibigrid
+  # Hostname or IP address of the Bibigrid server.
   https: False
+  # Set to True if Bibigrid should use HTTPS, otherwise False.
   localDnsLookup: False
+  # Set to True if Bibigrid should perform local DNS lookup, otherwise False.
   sub_network: portalexternalsubnetwork
+  # Name or identifier of the openstack subnetwork.
   modes:
     - slurm
-  ansibleGalaxyRoles:
-    - name: autoscaling
-      hosts: master
-      git: https://github.com/patricS4/autoscaling-config-ansible
 
+
+# FORC
 forc:
+  activated: True
+  # If Forc ist activated  -- defaults to True
   forc_url: https://proxy-dev.bi.denbi.de:5000/
+  # URL for FORC.
   forc_access_url: https://proxy-dev.bi.denbi.de/
+  # Access URL for FORC - optional - default is forc_url without port.
   github_playbooks_repo: https://github.com/deNBI/resenvs/archive/refs/heads/staging.zip
-  forc_security_group_id: 9a08eecc-d9a5-405b-aeda-9d4180fc94d6
-
-cloud_site: bielefeld
+  # GitHub repository URL for FORC playbooks.

simple_vm_client/forc_connector/forc_connector.py

@@ -33,15 +33,14 @@ def __init__(self, config_file: str):
 
         self.FORC_URL: str = ""  # type: ignore
         self.FORC_ACCESS_URL: str = ""  # type: ignore
-        self.FORC_REMOTE_ID: str = ""  # type: ignore
         self.GITHUB_PLAYBOOKS_REPO: str = ""  # type: ignore
         self.REDIS_HOST: str = ""  # type: ignore
         self.REDIS_PORT: int = None  # type: ignore
+        self.FORC_API_KEY: str = ""
         self.redis_pool: redis.ConnectionPool = None  # type: ignore
         self.redis_connection: redis.Redis.connection_pool = None
         self._active_playbooks: dict[str, Playbook] = {}
         self.load_config(config_file=config_file)
-        self.load_env()
         self.connect_to_redis()
         self.template = Template(
             github_playbook_repo=self.GITHUB_PLAYBOOKS_REPO,
@@ -53,15 +52,24 @@ def load_config(self, config_file: str) -> None:
         logger.info("Load config file: FORC")
         with open(config_file, "r") as ymlfile:
             cfg = yaml.load(ymlfile, Loader=yaml.SafeLoader)
+            # Check if "bibigrid" key is present in the loaded YAML
+            self.REDIS_HOST = cfg["redis"]["host"]
+            self.REDIS_PORT = cfg["redis"]["port"]
+            if "forc" not in cfg:
+                # Optionally, you can log a message or take other actions here
+                logger.info("Forc configuration not found. Skipping.")
+                return
+            if not cfg["forc"].get("activated", True):
+                logger.info("Forc Config available but deactivated. Skipping..")
+                return
             self.FORC_URL = cfg["forc"]["forc_url"]
             url_components = urlparse(cfg["forc"]["forc_url"])
             path = url_components.netloc.split(":")[0]
 
             self.FORC_ACCESS_URL = f"{url_components.scheme}://{path}/"
-            self.FORC_REMOTE_ID = cfg["forc"]["forc_security_group_id"]
             self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"]
-            self.REDIS_HOST = cfg["redis"]["host"]
-            self.REDIS_PORT = cfg["redis"]["port"]
+
+        self.load_env()
 
     def connect_to_redis(self) -> None:
         logger.info("Connect to redis")

simple_vm_client/forc_connector/template/template.py

@@ -84,8 +84,10 @@ def __init__(self, github_playbook_repo: str, forc_url: str, forc_api_key: str):
         self.GITHUB_PLAYBOOKS_REPO = github_playbook_repo
         self.FORC_URL = forc_url
         self.FORC_API_KEY = forc_api_key
-        self.TEMPLATES_URL = f"{self.FORC_URL}templates"
-        self.BACKENDS_URL = f"{self.FORC_URL}backends"
+        if not self.FORC_URL:
+            logger.info("No FORC URL defined. Skipping Forc...")
+        self.TEMPLATES_URL = f"{self.FORC_URL}templates" if self.FORC_URL else ""
+        self.BACKENDS_URL = f"{self.FORC_URL}backends" if self.FORC_URL else ""
         self.BACKENDS_BY_OWNER_URL = f"{self.BACKENDS_URL}/byOwner"
         self.BACKENDS_BY_TEMPLATE_URL = f"{self.BACKENDS_URL}/byTemplate"
         self._forc_allowed: dict[str, list[str]] = {}
@@ -158,8 +160,8 @@ def _process_template_metadata(
                 ] = template_metadata
 
     def update_playbooks(self) -> None:
-        if self.GITHUB_PLAYBOOKS_REPO is None:
-            logger.error(
+        if not self.GITHUB_PLAYBOOKS_REPO:
+            logger.warning(
                 "Github playbooks repo URL is None. Aborting download of playbooks."
             )
             return
@@ -179,18 +181,19 @@ def update_playbooks(self) -> None:
         logger.info(f"Allowed Forc {self._forc_allowed}")
 
     def _get_forc_templates(self) -> list[dict]:
-        try:
-            response = requests.get(
-                self.TEMPLATES_URL,
-                timeout=(30, 30),
-                headers={"X-API-KEY": self.FORC_API_KEY},
-                verify=True,
-            )
-            response.raise_for_status()  # Raise HTTPError for bad responses
-            return response.json()
-        except requests.RequestException as e:
-            logger.exception(f"Error while fetching FORC templates: {e}")
-            return []
+        if self.TEMPLATES_URL:
+            try:
+                response = requests.get(
+                    self.TEMPLATES_URL,
+                    timeout=(30, 30),
+                    headers={"X-API-KEY": self.FORC_API_KEY},
+                    verify=True,
+                )
+                response.raise_for_status()  # Raise HTTPError for bad responses
+                return response.json()
+            except requests.RequestException as e:
+                logger.exception(f"Error while fetching FORC templates: {e}")
+        return []
 
     def cross_check_forc_image(self, tags: list[str]) -> bool:
         try:

simple_vm_client/forc_connector/template/test_templates.py

@@ -266,11 +266,11 @@ def test_update_loaded_templates(self, mock_isdir, mock_listdir):
         expected_templates = ["template1", "template2"]
         self.assertEqual(template._all_templates, expected_templates)
 
-    @patch("simple_vm_client.forc_connector.template.template.logger.error")
-    def test_update_playbooks_no_github_repo(self, mock_logger_error):
+    @patch("simple_vm_client.forc_connector.template.template.logger.warning")
+    def test_update_playbooks_no_github_repo(self, mock_logger_warning):
         template = self.init_template()
         template.update_playbooks()
-        mock_logger_error.assert_called_once_with(
+        mock_logger_warning.assert_called_once_with(
             "Github playbooks repo URL is None. Aborting download of playbooks."
         )