Merge pull request #438 from deNBI/feat/unittest

Feat/unittest

.coveragerc

@@ -0,0 +1,18 @@
+[run]
+omit =
+    simple_vm_client/VirtualMachineService.py
+    simple_vm_client/constants.py
+    simple_vm_client/test_openstack_connector.py
+    simple_vm_client/ttypes.py
+    simple_vm_client/forc_connector/template/test_templates.py
+    simple_vm_client/util/logger.py
+    simple_vm_client/forc_connector/test_forc_connector.py
+    simple_vm_client/VirtualMachineServer.py
+
+    check_env.py
+
+[report]
+exclude_lines =
+    pragma: no cover
+    raise NotImplementedError
+    if __name__ == .__main__.:

.github/workflows/coverage.yml

@@ -0,0 +1,33 @@
+# This workflow will install dependencies, create coverage tests and run Pytest Coverage Comment
+# For more information see: https://github.com/MishaKav/pytest-coverage-comment/
+name: pytest-coverage-comment
+on:
+  pull_request:
+    branches:
+      - '*'
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.11
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 pytest pytest-cov
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+
+      - name: Build coverage file
+        run: |
+          pytest --junitxml=pytest.xml | tee pytest-coverage.txt
+
+      - name: Pytest coverage comment
+        uses: MishaKav/pytest-coverage-comment@main
+        with:
+          pytest-coverage-path: ./pytest-coverage.txt
+          junitxml-path: ./pytest.xml

.pre-commit-config.yaml

@@ -9,10 +9,12 @@ repos:
       - id: end-of-file-fixer
       - id: check-yaml
 
-  - repo: https://github.com/psf/black
-    rev: 23.12.1
+  - repo: https://github.com/psf/black-pre-commit-mirror
+
+    rev: 23.12.0
     hooks:
       - id: black
+        language_version: python3.11
   - repo: https://github.com/sondrelg/pep585-upgrade
     rev: 'v1.0.1'  # Use the sha / tag you want to point at
     hooks:

Dockerfile

@@ -1,11 +1,24 @@
 FROM python:3.11.4-buster
-RUN  apt-get update -y
-RUN apt-get install -y build-essential
+
+RUN apt-get update -y \
+    && apt-get install -y build-essential \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
 WORKDIR /code
-ADD requirements.txt /code
+
+# Copy requirements and install them first to leverage Docker cache
+COPY requirements.txt /code
 RUN pip install -r requirements.txt
-ADD requirements.yml /code
-ADD ansible.cfg /etc/ansible/
+
+COPY requirements.yml /code
+COPY ansible.cfg /etc/ansible/
 RUN ansible-galaxy install -r requirements.yml
-ADD . /code
+
+# Copy the entire project
+COPY . /code
+
+# Set PYTHONPATH to include the project root
+ENV PYTHONPATH /code
+
 WORKDIR /code/simple_vm_client

Makefile

@@ -18,7 +18,6 @@ thrift_py: ## Builds python code from thrift file
 	thrift --gen py portal_client.thrift
 	cp -a gen-py/VirtualMachineService/. simple_vm_client
 	rm -rf gen-py
-	@echo Remember to fix the imports: for pip relative imports are needed, for others absolute imports
 
 dev-build: ## Build and Start the docker-compose.dev.yml
 	docker-compose -f docker-compose.dev.yml up --build

pytest.ini

@@ -0,0 +1,6 @@
+# pytest.ini
+
+[pytest]
+addopts = --cov=. --cov-config=.coveragerc
+
+# Add other configuration options as needed

setup.cfg

@@ -27,5 +27,3 @@ warn_unused_configs = True
 [coverage:run]
 include = simple_vm_client/*
 omit = *migrations*, *tests*
-plugins =
-    django_coverage_plugin

simple_vm_client/VirtualMachineHandler.py

@@ -5,29 +5,27 @@
 """
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
-
-from bibigrid_connector.bibigrid_connector import BibigridConnector
-from forc_connector.forc_connector import ForcConnector
-from openstack_connector.openstack_connector import OpenStackConnector
-from util import thrift_converter
-from util.logger import setup_custom_logger
-from VirtualMachineService import Iface
-
-if TYPE_CHECKING:
-    from ttypes import (
-        VM,
-        Backend,
-        ClusterInfo,
-        ClusterInstance,
-        CondaPackage,
-        Flavor,
-        Image,
-        PlaybookResult,
-        ResearchEnvironmentTemplate,
-        Snapshot,
-        Volume,
-    )
+
+from simple_vm_client.bibigrid_connector.bibigrid_connector import BibigridConnector
+from simple_vm_client.forc_connector.forc_connector import ForcConnector
+from simple_vm_client.openstack_connector.openstack_connector import OpenStackConnector
+from simple_vm_client.util import thrift_converter
+from simple_vm_client.util.logger import setup_custom_logger
+
+from .ttypes import (
+    VM,
+    Backend,
+    ClusterInfo,
+    ClusterInstance,
+    CondaPackage,
+    Flavor,
+    Image,
+    PlaybookResult,
+    ResearchEnvironmentTemplate,
+    Snapshot,
+    Volume,
+)
+from .VirtualMachineService import Iface
 
 logger = setup_custom_logger(__name__)
 
@@ -143,10 +141,13 @@ def get_server(self, openstack_id: str) -> VM:
         return server
 
     def get_servers(self) -> list[VM]:
-        serv = thrift_converter.os_to_thrift_servers(
-            openstack_servers=self.openstack_connector.get_servers()
-        )
-        return serv
+        servers = openstack_servers = self.openstack_connector.get_servers()
+        servers_full = []
+
+        for server in servers:
+            servers_full.append(self.forc_connector.get_playbook_status(server=server))
+        serv = thrift_converter.os_to_thrift_servers(openstack_servers=servers)
+        return servers_full
 
     def get_servers_by_ids(self, server_ids: list[str]) -> list[VM]:
         return thrift_converter.os_to_thrift_servers(

simple_vm_client/VirtualMachineServer.py

@@ -1,4 +1,3 @@
-import os
 import signal
 import ssl
 import sys
@@ -8,8 +7,9 @@
 from thrift.protocol import TBinaryProtocol
 from thrift.server import TServer
 from thrift.transport import TSocket, TSSLSocket, TTransport
-from VirtualMachineHandler import VirtualMachineHandler
-from VirtualMachineService import Processor
+
+from simple_vm_client.VirtualMachineHandler import VirtualMachineHandler
+from simple_vm_client.VirtualMachineService import Processor
 
 USERNAME = "OS_USERNAME"
 PASSWORD = "OS_PASSWORD"
@@ -72,15 +72,5 @@ def catch_shutdown(signal: int, frame: object) -> None:
     server.serve()
 
 
-def check_environment_variables(envs: list[str]) -> None:
-    def check_env(var: str) -> None:
-        if var not in os.environ:
-            click.echo(f"ERROR: There is no {var} set in environment.")
-            click.echo("Please make sure you have sourced your OpenStack rc file")
-            sys.exit()
-
-    list(map(lambda var: check_env(var), envs))
-
-
 if __name__ == "__main__":
     startServer()

simple_vm_client/__init__.py

@@ -1 +1 @@
-__all__ = ['ttypes', 'constants', 'VirtualMachineService']
+__all__ = ["ttypes", "constants", "VirtualMachineService"]

simple_vm_client/bibigrid_connector/bibigrid_connector.py

@@ -1,7 +1,8 @@
 import requests
 import yaml
-from ttypes import ClusterInfo, ClusterInstance
-from util.logger import setup_custom_logger
+
+from simple_vm_client.ttypes import ClusterInfo, ClusterInstance
+from simple_vm_client.util.logger import setup_custom_logger
 
 logger = setup_custom_logger(__name__)
 
@@ -53,27 +54,32 @@ def load_config_yml(self, config_file: str) -> None:
 
     def get_cluster_status(self, cluster_id: str) -> dict[str, str]:
         logger.info(f"Get Cluster {cluster_id} status")
+
         headers = {"content-Type": "application/json"}
         body = {"mode": "openstack"}
-        request_url = self._BIBIGRID_URL + "info/" + cluster_id
-        response = requests.get(
-            url=request_url,
-            json=body,
-            headers=headers,
-            verify=self._PRODUCTION,
-        )
-        logger.info(f"Cluster {cluster_id} status: {str(response.content)} ")
-        json_resp: dict[str, str] = response.json(strict=False)
-        try:
-            json_resp["log"] = str(json_resp["log"])
-        except Exception:
-            logger.info(f"No Logs for Cluster - {cluster_id}")
+        request_url = f"{self._BIBIGRID_URL}info/{cluster_id}"
+
         try:
-            json_resp["msg"] = str(json_resp["msg"])
-        except Exception:
-            logger.info(f"No msg for Cluster - {cluster_id}")
+            response = requests.get(
+                url=request_url,
+                json=body,
+                headers=headers,
+                verify=self._PRODUCTION,
+            )
+            response.raise_for_status()  # Raise an exception for HTTP errors (4xx and 5xx)
+            json_resp = response.json(strict=False)
+
+            # Convert log and msg keys to strings, handling the case where they might not exist
+            json_resp["log"] = str(json_resp.get("log", ""))
+            json_resp["msg"] = str(json_resp.get("msg", ""))
 
-        return json_resp
+            logger.info(f"Cluster {cluster_id} status: {json_resp}")
+
+            return json_resp
+
+        except requests.RequestException as e:
+            logger.exception("Error while getting Cluster status")
+            return {"error": str(e)}
 
     def get_cluster_info(self, cluster_id: str) -> ClusterInfo:
         logger.info(f"Get Cluster info from {cluster_id}")
@@ -127,7 +133,7 @@ def is_bibigrid_available(self) -> bool:
                 logger.error(f"Bibigrid returned status code {response.status_code}")
                 return False
 
-        except requests.RequestException as e:
+        except requests.RequestException:
             logger.exception("Error while checking Bibigrid availability")
             return False