Merge branch 'main' into fix/76-dry-run-error

.github/workflows/pre-commit.yml → .github/workflows/on-push-pre-commit.yml

@@ -1,4 +1,4 @@
-name: pre-commit
+name: py:pre-commit
 
 on:
   pull_request:
@@ -11,5 +11,5 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - uses: actions/setup-python@v3
-    - run: python -m pip install .[tests]
+    - run: pip install .[tests]
     - uses: pre-commit/[email protected]

.github/workflows/on-push-test.yml

@@ -0,0 +1,52 @@
+name: py:test
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  codeql-analysis:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v3
+    - name: Set up Python
+      id: setup-python
+      uses: actions/setup-python@v3
+    - name: Get the Python path
+      id: get-python-path
+      run: echo "python-path=`which python`"
+    - name: Install dependencies
+      run: |-
+        pip install -r ./requirements.txt
+        pip install pylint --upgrade
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: python
+        queries: security-extended
+    - name: Perform CodeQL analysis
+      env:
+        CODEQL_PYTHON: ${{ steps.get-python-path.outputs.python-path }}
+      uses: github/codeql-action/analyze@v3
+
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ "3.9", "3.10", "3.11", "3.12" ]
+
+    steps:
+    - name: Checkout commit
+      uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |-
+        pip install -r ./requirements.txt
+    - name: Execute tests
+      run: pytest

Dockerfile

@@ -27,6 +27,7 @@ LABEL org.opencontainers.image.source="https://github.com/SovereignCloudStack/ro
 
 ARG ROOKIFY_VERSION=0.0.0.dev1
 ENV ROOKIFY_VERSION=$ROOKIFY_VERSION
+ENV PYTHONPATH="${PYTHONPATH}:/app/rookify/src"
 
 WORKDIR /app/rookify
 

Makefile

@@ -4,12 +4,8 @@ COLOUR_BLUE=\033[0;34m
 COLOUR_END=\033[0m
 
 .DEFAULT_GOAL:=help
-SHELL := /bin/bash
+SHELL:=/bin/bash
 
-# Get needed paths and information from locally installed librados
-export RADOSLIB_VERSION := 2.0.0
-export GENERAL_LIB_LOCATION := ${shell pip show rados | grep -oP "(?<=Location: ).*"}
-export RADOSLIB_INSTALLED_VERSION := ${shell pip show rados | grep Version | awk '{print $$2}'}
 
 ## checking if docker, or podman should be used. Podman is preferred.
 ifeq ($(shell command -v podman 2> /dev/null),)
@@ -18,27 +14,41 @@ else
 	CONTAINERCMD=podman
 endif
 
+# Checking if python exists
+ifneq (, $(shell command -v python))
+    $(info Python is installed as 'python')
+    PYTHON := $(shell command -v python)
+else ifneq (, $(shell command -v python3))
+    $(info Python3 is installed as 'python3')
+    PYTHON := $(shell command -v python3)
+else
+    $(error Neither python nor python3 is installed)
+endif
+
 ## Export default rookify version
-export ROOKIFY_VERSION ?= "0.0.0.dev1"
+export ROOKIFY_VERSION?=0.0.0.dev1
 
 .PHONY: help
 help: ## Display this help message
-	@echo -e '${COLOUR_RED}Usage: make <command>${COLOUR_END}'
+	@echo -e '\n${COLOUR_BLUE}ROOKIFY MAKEFILE${COLOUR_BLUE}'
+	@echo -e '\n${COLOUR_RED}Usage: make <command>${COLOUR_END}'
 	@cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]'  | \
 	    awk -F ':.*?## ' 'NF==2 {printf "  %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}'
+	@echo -e '${COLOUR_RED}OSISM helperscript usage: make <command>${COLOUR_END}'
+	@cat $(MAKEFILE_LIST) | grep '^[a-zA-Z]'  | \
+	    awk -F ':.*?#osism# ' 'NF==2 {printf "  %-26s%s\n\n", $$1, "${COLOUR_GREEN}"$$2"${COLOUR_END}"}'
 
 .PHONY: setup
-setup: setup-pre-commit check-radoslib setup-venv ## Setup the pre-commit environment and then the venv environment
+setup: check-radoslib setup-venv setup-pre-commit ## Setup the pre-commit environment and then the venv environment
 
 .PHONY: setup-pre-commit
 setup-pre-commit:
-	pip install --user pre-commit && pre-commit install
+	./.venv/bin/pip install --user pre-commit && ./.venv/bin/python -m pre_commit install
 
 .PHONY: setup-venv
 setup-venv:
-	python -m venv --system-site-packages ./.venv && \
-	source ./.venv/bin/activate && \
-	pip install -r requirements.txt
+	${PYTHON} -m venv --system-site-packages ./.venv && \
+	./.venv/bin/pip install -r requirements.txt
 
 .PHONY: run-precommit
 run-precommit: ## Run pre-commit to check if all files running through
@@ -50,33 +60,30 @@ update-requirements: ## Update the requirements.txt with newer versions of pip p
 	pip freeze -l > requirements.txt
 
 .PHONY: check-radoslib
+export RADOSLIB_VERSION:=2.0.0
 check-radoslib: ## Checks if radoslib is installed and if it contains the right version
-	@if [ -z "${GENERAL_LIB_LOCATION}" ]; then \
-		echo -e "${COLOUR_RED}ERROR: 'rados' library not found. Please make sure it's installed.${COLOUR_END}"; \
-		exit 1; \
-	else \
-		echo -e "GENERAL_LIB_LOCATION: $(GENERAL_LIB_LOCATION)"; \
-	fi
-	@if [ "${RADOSLIB_INSTALLED_VERSION}" != "${RADOSLIB_VERSION}" ]; then \
-		echo -e "${COLOUR_RED}ERROR: Incorrect version of 'rados' library found. Expected version $(RADOSLIB_VERSION), found $$RADOSLIB_INSTALLED_VERSION.${COLOUR_END}"; \
-		exit 1; \
-	else \
-		echo -e "RADOSLIB_INSTALLED_VERSION: $(RADOSLIB_INSTALLED_VERSION)"; \
-	fi
+	# Get needed paths and information from locally installed librados
+	./scripts/check_local_rados_lib_installation.sh ${RADOSLIB_VERSION}
+
+.PHONY: build-local-rookify
+build-local-rookify: ## This builds rookify into .venv/bin/rookify
+	source .venv/bin/activate && pip install -e .
+
+.PHONY: build-container
+build-container: ## Build container from Dockerfile, add e.g. ROOKIFY_VERSION=0.0.1 to specify the version. Default value is 0.0.0.dev1
+	${CONTAINERCMD} build --build-arg ROOKIFY_VERSION=$(ROOKIFY_VERSION) --target rookify -t rookify:latest -f Dockerfile .
 
 .PHONY: run-local-rookify
 run-local-rookify: ## Runs rookify in the local development environment (requires setup-venv)
-	$(eval PYTHONPATH="${PYTHONPATH}:$(pwd)/src")
-	source ./.venv/bin/activate && \
-	cd src && python -m rookify
+	source ./.venv/bin/activate && pip install -e . && rookify
 
 .PHONY: run-rookify
 run-rookify: ## Runs rookify in the container
 	docker exec -it rookify-dev /app/rookify/.venv/bin/rookify
 
-.PHONY: build-container
-build-container: ## Build container from Dockerfile, add e.g. ROOKIFY_VERSION=0.0.1 to specify the version. Default value is 0.0.0.dev1
-	${CONTAINERCMD} build --build-arg ROOKIFY_VERSION=$(ROOKIFY_VERSION) --target rookify -t rookify:latest -f Dockerfile .
+.PHONY: get-testbed-configs-for-rookify-testing
+get-testbed-configs-for-rookify-testing: ## Gets the needed config (like .kube, /etc/ceph and so on) from the testbed
+	bash ./scripts/get_configs_from_testbed.sh
 
 .PHONY: run-tests-locally
 run-tests-locally: ## Runs the tests in the tests directory. NB: check that your local setup is connected through vpn to the testbed!
@@ -101,3 +108,11 @@ down: ## Remove the containers as setup by docker-compose.yml
 .PHONY: up
 up: ## Sets up the container as specified in docker-compose.yml and opens a bash terminal
 	${CONTAINERCMD} compose up -d
+
+##
+# Add osism specific scripts below here (so they appear below helper header)
+##
+
+.PHONY: get-config
+get-config: #osism# Gets configuration files from the OSISM testbed
+	./scripts/osism/get_osism_configs_from_testbed.sh

mock_src/rados.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+
+pass

pyproject.toml

@@ -26,4 +26,4 @@ classifiers = [
 Homepage = "https://scs.community"
 
 [tool.pytest.ini_options]
-pythonpath = [ "src" ]
+pythonpath = [ "src", "mock_src" ]

requirements.txt

@@ -1,17 +1,17 @@
 bcrypt==4.1.2
 cachetools==5.3.2
-certifi==2024.2.2
+certifi==2024.7.4
 cffi==1.16.0
 charset-normalizer==3.3.2
-cryptography==42.0.4
+cryptography==43.0.1
 dill==0.3.8
 decorator==5.1.1
 Deprecated==1.2.14
 fabric==3.2.2
 google-auth==2.28.1
-idna==3.6
+idna==3.7
 invoke==2.2.0
-Jinja2==3.1.3
+Jinja2==3.1.4
 kubernetes==29.0.0
 MarkupSafe==2.1.5
 oauthlib==3.2.2
@@ -22,13 +22,15 @@ pycparser==2.21
 PyNaCl==1.5.0
 python-dateutil==2.8.2
 PyYAML==6.0.1
-requests==2.31.0
+requests==2.32.2
 requests-oauthlib==1.3.1
 rsa==4.9
 six==1.16.0
 structlog==24.1.0
 transitions==0.9.0
-urllib3==2.2.1
+urllib3==2.2.2
 yamale==5.1.0
 websocket-client==1.7.0
 wrapt==1.16.0
+pytest==8.0.2
+pytest-structlog==1.1

scripts/check_local_rados_lib_installation.sh

@@ -0,0 +1,29 @@
+#!/bin/env bash
+
+# Check if a version argument is provided
+if [ $# -eq 1 ]; then
+    RADOSLIB_VERSION="$1"
+else
+    # Default version if no argument is provided
+    RADOSLIB_VERSION="2.0.0"
+fi
+
+# Get the location of the installed rados library
+GENERAL_LIB_LOCATION=$(pip show rados | grep -oP "(?<=Location: ).*")
+
+# Get the installed version of the rados library
+RADOSLIB_INSTALLED_VERSION=$(pip show rados | grep Version | awk '{print $2}')
+
+# Check if the rados library is installed
+if [ -z "$GENERAL_LIB_LOCATION" ]; then
+    echo -e "\033[0;31mERROR: 'rados' library not found. Please make sure it's installed.\033[0m"
+    exit 1
+fi
+
+# Check if the installed version matches the expected version
+if [ "$RADOSLIB_INSTALLED_VERSION" != "$RADOSLIB_VERSION" ]; then
+    echo -e "\033[0;31mERROR: 'rados' library version $RADOSLIB_INSTALLED_VERSION does not match the expected version $RADOSLIB_VERSION.\033[0m"
+    exit 1
+else
+    echo -e "\033[0;32m'rados' library version $RADOSLIB_INSTALLED_VERSION is correct.\033[0m"
+fi

src/rookify/__main__.py

@@ -1,33 +1,67 @@
 # -*- coding: utf-8 -*-
 
+import sys
+import argparse
 from argparse import ArgumentParser
+from typing import Any, Dict
 from .modules import load_modules
 from .modules.machine import Machine
+from .modules.module import ModuleHandler
 from .logger import configure_logging, get_logger
 from .yaml import load_config
 
 
-def main() -> None:
+def parse_args(args: list[str]) -> argparse.Namespace:
+    # Putting args-parser in seperate function to make this testable
     arg_parser = ArgumentParser("Rookify")
+
+    # --dry-run option
     arg_parser.add_argument("--dry-run", action="store_true", dest="dry_run_mode")
-    args = arg_parser.parse_args()
+
+    arg_parser.add_argument(
+        "--show-states",
+        action="store_true",
+        dest="show_states",
+        help="Show states of the modules.",
+    )
+
+    return arg_parser.parse_args(args)
+
+
+def main() -> None:
+    args = parse_args(sys.argv[1:])
 
     # Load configuration file
     try:
-        config = load_config("config.yaml")
+        config: Dict[str, Any] = load_config("config.yaml")
     except FileNotFoundError as err:
         raise SystemExit(f"Could not load config: {err}")
 
     # Configure logging
     try:
-        configure_logging(config["logging"])
+        if args.show_states is True:
+            configure_logging(
+                {"level": "ERROR", "format": {"renderer": "console", "time": "iso"}}
+            )
+        else:
+            configure_logging(config["logging"])
     except Exception as e:
         raise SystemExit(f"Error configuring logging: {e}")
 
+    # Get Logger
     log = get_logger()
-    log.debug("Executing Rookify")
+
+    log.info("Executing Rookify ...")
 
     machine = Machine(config["general"].get("machine_pickle_file"))
+
     load_modules(machine, config)
 
-    machine.execute(args.dry_run_mode)
+    if args.show_states is True:
+        ModuleHandler.show_states(machine, config)
+    else:
+        machine.execute(dry_run_mode=args.dry_run_mode)
+
+
+if __name__ == "__main__":
+    main()