diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml new file mode 100644 index 000000000..0b983ee1a --- /dev/null +++ b/.github/workflows/linting.yml @@ -0,0 +1,20 @@ +name: linting +on: [push] +jobs: + linting-job: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v4 + with: + python-version: 3.8 + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements-dev.txt + - name: ansible_lint + run: ansible-lint resources/playbook/roles/bibigrid/tasks/main.yml + - name: pylint_lint + run: pylint bibigrid \ No newline at end of file diff --git a/.github/workflows/publish_docker_dev.yml b/.github/workflows/publish_docker_dev.yml deleted file mode 100644 index 66702dd3c..000000000 --- a/.github/workflows/publish_docker_dev.yml +++ /dev/null @@ -1,30 +0,0 @@ -name: Publish DEV Docker -on: - push: - branches: - - 'development' -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@master - - name: Set up JDK 1.11 - uses: actions/setup-java@v1 - with: - java-version: '11.0.5' - - name: Build mvn bibigrid - run: | - mvn -P openstack clean install - - name: Build rest api mvn - run: | - cd bibigrid-light-rest-4j/ - mvn clean install -Prelease -DskipTests - - name: Publish to Registry - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: bibiserv/bibigrid - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - dockerfile: Dockerfile - workdir: bibigrid-light-rest-4j - tags: dev diff --git a/.github/workflows/publish_docker_release.yml b/.github/workflows/publish_docker_release.yml deleted file mode 100644 index 8b1e5080d..000000000 --- a/.github/workflows/publish_docker_release.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: Publish RELEASE Docker -on: - push: - tags: - - '*' -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Get the version - id: get_version - run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} - - uses: actions/checkout@master - - name: Set up JDK 1.11 - uses: actions/setup-java@v1 - with: - java-version: '11.0.5' - - name: Build mvn bibigrid - run: | - mvn -P openstack clean install - - name: Build rest api mvn - run: | - cd bibigrid-light-rest-4j/ - mvn clean install -Prelease -DskipTests - - name: Publish to Registry - uses: elgohr/Publish-Docker-Github-Action@master - with: - name: bibiserv/bibigrid - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - dockerfile: Dockerfile - workdir: bibigrid-light-rest-4j - tags: bibigrid-rest-${{ steps.get_version.outputs.VERSION }} diff --git a/.gitignore b/.gitignore index bc44901af..bea74e838 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # complete idea .idea/ +.run/ # variable resources resources/playbook/site.yml diff --git a/.pylintrc b/.pylintrc index d3d3a2306..42d86e716 100644 --- a/.pylintrc +++ b/.pylintrc @@ -82,7 +82,7 @@ persistent=no # Minimum Python version to use for version dependent checks. Will default to # the version used to run pylint. -py-version=3.10 +#py-version=3.10 # Discover python modules and packages in the file system subtree. recursive=no diff --git a/README.md b/README.md index f48ce822d..d3eba843e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # BiBiGrid -BiBiGrid is a cloud cluster creation and management framework for OpenStack (and more providers in the future). +BiBiGrid is a cloud cluster creation and management framework for OpenStack +(and more providers in the future). + +BiBiGrid uses Ansible to configure standard Ubuntu 20.04/22.04 LTS as +well as Debian 11 cloud images. Depending on your configuration BiBiGrid +can set up an HCP cluster for grid computing (Slurm Workload Manager, +a shared filesystem (on local discs and attached volumes), +a cloud IDE for writing, running and debugging (Theia Web IDE) and many more. + > **Note** > The latest version is currently work in progress. Future changes are likely. @@ -15,7 +23,7 @@ However, if you are already quite experienced with *OpenStack* and the previous might be just what you need.
- Brief, technical BiBiGrid2 overview + Brief, technical BiBiGrid overview ### How to configure a cluster? #### Configuration File: bibigrid.yml @@ -33,7 +41,7 @@ The configuration template [bibigrid.yml](bibigrid.yml) contains many helpful co To access the cloud, authentication information is required. You can download your `clouds.yaml` from OpenStack. -Your `clouds.yaml` is to be placed in `~/.config/bibigrid/` and will be loaded by BiBiGrid2 on execution. +Your `clouds.yaml` is to be placed in `~/.config/bibigrid/` and will be loaded by BiBiGrid on execution. [You need more details?](documentation/markdown/features/cloud_specification_data.md) @@ -49,30 +57,30 @@ a region, an availability zone, an sshUser (most likely ubuntu) and a subnet. You probably also want at least one worker with a valid type, image and count. 4. If your cloud provider runs post-launch services, you need to set the `waitForServices` key appropriately which expects a list of services to wait for. -5. Create a virtual environment from `bibigrid2/requirements.txt`. +5. Create a virtual environment from `bibigrid/requirements.txt`. See [here](https://www.akamai.com/blog/developers/how-building-virtual-python-environment) for more detailed info. 6. Take a look at [First execution](#first-execution) #### First execution Before follow the steps described at [Preparation](#preparation). -After cloning the repository navigate to `bibigrid2`. -In order to execute BiBiGrid2 source the virtual environment created during [preparation](#preparation). -Take a look at BiBiGrid2's [Command Line Interface](documentation/markdown/features/CLI.md) +After cloning the repository navigate to `bibigrid`. +In order to execute BiBiGrid source the virtual environment created during [preparation](#preparation). +Take a look at BiBiGrid's [Command Line Interface](documentation/markdown/features/CLI.md) if you want to explore for yourself. A first execution run through could be: 1. `./bibigrid.sh -i [path-to-bibigrid.yml] -ch`: checks the configuration 2. `./bibigrid.sh -i 'bibigrid.yml -i [path-to-bibigrid.yml] -c'`: creates the cluster (execute only if check was successful) -3. Use **BiBiGrid2's create output** to investigate the created cluster further. Especially connecting to the ide might be helpful. +3. Use **BiBiGrid's create output** to investigate the created cluster further. Especially connecting to the ide might be helpful. Otherwise, connect using ssh. 4. While in ssh try `sinfo` to printing node info 5. Run `srun -x $(hostname) hostname` to power up a worker and get its hostname. 6. Run `sinfo` again to see the node powering up. After a while it will be terminated again. -7. Use the terminate command from **BiBiGrid2's create output** to shut down the cluster again. +7. Use the terminate command from **BiBiGrid's create output** to shut down the cluster again. All floating-ips used will be released. -Great! You've just started and terminated your first cluster using BiBiGrid2! +Great! You've just started and terminated your first cluster using BiBiGrid!
@@ -87,21 +95,21 @@ Some quotas can currently not be checked by bibigrid. **Whenever you contact a developer, please send your logfile along.** # Documentation -If you would like to learn more about BiBiGrid2 please follow a fitting link: -- [BiBiGrid2 Features](documentation/markdown/bibigrid_feature_list.md) -- [Software used by BiBiGrid2](documentation/markdown/bibigrid_software_list.md) +If you would like to learn more about BiBiGrid please follow a fitting link: +- [BiBiGrid Features](documentation/markdown/bibigrid_feature_list.md) +- [Software used by BiBiGrid](documentation/markdown/bibigrid_software_list.md)
- Differences to BiBiGrid1 + Differences to old Java BiBiGrid -* BiBiGrid2 no longer uses RC- but cloud.yaml-files for cloud-specification data. Environment variables are no longer used (or supported). +* BiBiGrid no longer uses RC- but cloud.yaml-files for cloud-specification data. Environment variables are no longer used (or supported). See [Cloud Specification Data](documentation/markdown/features/cloud_specification_data.md). -* BiBiGrid2 has a largely reworked configurations file, because BiBiGrid2 core supports multiple providers this step was necessary. +* BiBiGrid has a largely reworked configurations file, because BiBiGrid core supports multiple providers this step was necessary. See [Configuration](documentation/markdown/features/configuration.md) -* BiBiGrid2 currently only implements the provider OpenStack. -* BiBiGrid2 only starts the master and will dynamically start workers using slurm when they are needed. +* BiBiGrid currently only implements the provider OpenStack. +* BiBiGrid only starts the master and will dynamically start workers using slurm when they are needed. Workers are powered down once they are not used for a longer period. -* BiBiGrid2 lays the foundation for clusters that are spread over multiple providers, but Hybrid Clouds aren't fully implemented yet. +* BiBiGrid lays the foundation for clusters that are spread over multiple providers, but Hybrid Clouds aren't fully implemented yet.
# Development @@ -112,5 +120,5 @@ Workers are powered down once they are not used for a longer period. ## On implementing concrete providers New concrete providers can be implemented very easily. Just copy the `provider.py` file and implement all methods for your cloud-provider. Also inherit from the `provider` class. After that add your provider to the providerHandler lists; giving it a associated name for the -configuration files. By that, your provider is automatically added to BiBiGrid2's tests and regular execution. By testing +configuration files. By that, your provider is automatically added to BiBiGrid's tests and regular execution. By testing your provider first, you will see whether all provider methods are implemented as expected. \ No newline at end of file diff --git a/bibigrid.sh b/bibigrid.sh index 7739c57ad..80fb82ce4 100755 --- a/bibigrid.sh +++ b/bibigrid.sh @@ -1 +1 @@ -python3 -m bibigrid2.core.startup "$@" \ No newline at end of file +python3 -m bibigrid.core.startup "$@" \ No newline at end of file diff --git a/bibigrid2/core/actions/check.py b/bibigrid/core/actions/check.py similarity index 70% rename from bibigrid2/core/actions/check.py rename to bibigrid/core/actions/check.py index 41797ec18..144eafbe0 100644 --- a/bibigrid2/core/actions/check.py +++ b/bibigrid/core/actions/check.py @@ -1,14 +1,14 @@ """ -Module that acts as a wrapper and uses validateConfiguration to validate given configuration +Module that acts as a wrapper and uses validate_configuration to validate given configuration """ import logging -from bibigrid2.core.utility import validate_configuration +from bibigrid.core.utility import validate_configuration LOG = logging.getLogger("bibigrid") def check(configurations, providers): """ - Uses validateConfiguration to validate given configuration. + Uses validate_configuration to validate given configuration. :param configurations: list of configurations (dicts) :param providers: list of providers :return: diff --git a/bibigrid2/core/actions/create.py b/bibigrid/core/actions/create.py similarity index 96% rename from bibigrid2/core/actions/create.py rename to bibigrid/core/actions/create.py index bd24ac3fe..609f638bc 100644 --- a/bibigrid2/core/actions/create.py +++ b/bibigrid/core/actions/create.py @@ -12,15 +12,15 @@ import paramiko import yaml -from bibigrid2.core.actions import terminate_cluster -from bibigrid2.core.utility import ansible_configurator -from bibigrid2.core.utility import id_generation -from bibigrid2.core.utility.handler import ssh_handler -from bibigrid2.core.utility.paths import ansible_resources_path as aRP -from bibigrid2.core.utility.paths import bin_path as biRP -from bibigrid2.models import exceptions -from bibigrid2.models import return_threading -from bibigrid2.models.exceptions import ExecutionException +from bibigrid.core.actions import terminate_cluster +from bibigrid.core.utility import ansible_configurator +from bibigrid.core.utility import id_generation +from bibigrid.core.utility.handler import ssh_handler +from bibigrid.core.utility.paths import ansible_resources_path as aRP +from bibigrid.core.utility.paths import bin_path as biRP +from bibigrid.models import exceptions +from bibigrid.models import return_threading +from bibigrid.models.exceptions import ExecutionException PREFIX = "bibigrid" SEPARATOR = "-" @@ -350,7 +350,7 @@ def print_cluster_start_info(self): """ Prints helpful cluster-info: SSH: How to connect to master via SSH - Terminate: What bibigrid2 command is needed to terminate the created cluster + Terminate: What bibigrid command is needed to terminate the created cluster Detailed cluster info: How to print detailed info about the created cluster :return: """ diff --git a/bibigrid2/core/actions/ide.py b/bibigrid/core/actions/ide.py similarity index 98% rename from bibigrid2/core/actions/ide.py rename to bibigrid/core/actions/ide.py index d1877a826..16f42bb21 100644 --- a/bibigrid2/core/actions/ide.py +++ b/bibigrid/core/actions/ide.py @@ -12,7 +12,7 @@ import webbrowser import sshtunnel -from bibigrid2.core.utility.handler import cluster_ssh_handler +from bibigrid.core.utility.handler import cluster_ssh_handler DEFAULT_IDE_WORKSPACE = "${HOME}" REMOTE_BIND_ADDRESS = 8181 diff --git a/bibigrid2/core/actions/list_clusters.py b/bibigrid/core/actions/list_clusters.py similarity index 99% rename from bibigrid2/core/actions/list_clusters.py rename to bibigrid/core/actions/list_clusters.py index 58f9924ae..07e0173f5 100644 --- a/bibigrid2/core/actions/list_clusters.py +++ b/bibigrid/core/actions/list_clusters.py @@ -7,7 +7,7 @@ import pprint import re -from bibigrid2.core.actions import create +from bibigrid.core.actions import create SERVER_REGEX = re.compile(r"^bibigrid-((master)-([a-zA-Z0-9]+)|(worker|vpnwkr)\d+-([a-zA-Z0-9]+)-\d+)$") LOG = logging.getLogger("bibigrid") diff --git a/bibigrid2/core/actions/terminate_cluster.py b/bibigrid/core/actions/terminate_cluster.py similarity index 98% rename from bibigrid2/core/actions/terminate_cluster.py rename to bibigrid/core/actions/terminate_cluster.py index 67f744dc8..fbd5bcdda 100644 --- a/bibigrid2/core/actions/terminate_cluster.py +++ b/bibigrid/core/actions/terminate_cluster.py @@ -7,7 +7,7 @@ import os import re -from bibigrid2.core.actions import create +from bibigrid.core.actions import create LOG = logging.getLogger("bibigrid") def terminate_cluster(cluster_id, providers, debug=False): @@ -83,7 +83,7 @@ def delete_keypairs(provider, tmp_keyname): """ Deletes keypairs from all provider @param provider: provider to delete keypair from - @param tmp_keyname: BiBiGrid2 keyname + @param tmp_keyname: BiBiGrid keyname @return: True if keypair was deleted """ LOG.info("Deleting Keypair on provider %s...", provider.cloud_specification['identifier']) @@ -98,7 +98,7 @@ def delete_keypairs(provider, tmp_keyname): def delete_local_keypairs(tmp_keyname): """ Deletes local keypairs of a cluster - @param tmp_keyname: BiBiGrid2 keyname + @param tmp_keyname: BiBiGrid keyname @return: Returns true if at least one local keyfile (pub or private) was found """ success = False diff --git a/bibigrid2/core/actions/update.py b/bibigrid/core/actions/update.py similarity index 75% rename from bibigrid2/core/actions/update.py rename to bibigrid/core/actions/update.py index 091e39300..90578de7a 100644 --- a/bibigrid2/core/actions/update.py +++ b/bibigrid/core/actions/update.py @@ -4,11 +4,11 @@ import logging -from bibigrid2.core.utility import ansible_commands as aC -from bibigrid2.core.utility.handler import ssh_handler -from bibigrid2.core.utility.paths import ansible_resources_path as aRP -from bibigrid2.core.utility.paths import bin_path as biRP -from bibigrid2.core.utility.handler import cluster_ssh_handler +from bibigrid.core.utility import ansible_commands as aC +from bibigrid.core.utility.handler import ssh_handler +from bibigrid.core.utility.paths import ansible_resources_path as aRP +from bibigrid.core.utility.paths import bin_path as biRP +from bibigrid.core.utility.handler import cluster_ssh_handler LOG = logging.getLogger("bibigrid") diff --git a/bibigrid2/core/actions/version.py b/bibigrid/core/actions/version.py similarity index 100% rename from bibigrid2/core/actions/version.py rename to bibigrid/core/actions/version.py diff --git a/bibigrid2/core/provider.py b/bibigrid/core/provider.py similarity index 100% rename from bibigrid2/core/provider.py rename to bibigrid/core/provider.py diff --git a/bibigrid2/core/startup.py b/bibigrid/core/startup.py similarity index 93% rename from bibigrid2/core/startup.py rename to bibigrid/core/startup.py index 7973d2ca5..69b9c5203 100755 --- a/bibigrid2/core/startup.py +++ b/bibigrid/core/startup.py @@ -10,11 +10,11 @@ import yaml -from bibigrid2.core.actions import check, create, ide, list_clusters, terminate_cluster, update, version -from bibigrid2.core.utility import command_line_interpreter -from bibigrid2.core.utility.handler import configuration_handler, provider_handler +from bibigrid.core.actions import check, create, ide, list_clusters, terminate_cluster, update, version +from bibigrid.core.utility import command_line_interpreter +from bibigrid.core.utility.handler import configuration_handler, provider_handler -LOGGING_HANDLER_LIST = [logging.StreamHandler(), logging.FileHandler("bibigrid2.log")] # stdout and to file +LOGGING_HANDLER_LIST = [logging.StreamHandler(), logging.FileHandler("bibigrid.log")] # stdout and to file VERBOSITY_LIST = [logging.WARNING, logging.INFO, logging.DEBUG] LOGGER_FORMAT = "%(asctime)s [%(levelname)s] %(message)s" @@ -38,7 +38,7 @@ def get_cluster_id_from_mem(): return None -def set_logger(verbosity): +def set_logger_verbosity(verbosity): """ Sets verbosity, format and handler. :param verbosity: level of verbosity @@ -48,7 +48,6 @@ def set_logger(verbosity): capped_verbosity = min(verbosity, len(VERBOSITY_LIST) - 1) # LOG.basicConfig(format=LOGGER_FORMAT, level=VERBOSITY_LIST[capped_verbosity], # handlers=LOGGING_HANDLER_LIST) - logging.basicConfig(format=LOGGER_FORMAT, handlers=LOGGING_HANDLER_LIST) log = logging.getLogger("bibigrid") log.setLevel(VERBOSITY_LIST[capped_verbosity]) @@ -126,9 +125,9 @@ def main(): Interprets command line, sets logger, reads configuration and runs selected action. Then exits. :return: """ - + logging.basicConfig(format=LOGGER_FORMAT, handlers=LOGGING_HANDLER_LIST) args = command_line_interpreter.interpret_command_line() - set_logger(args.verbose) + set_logger_verbosity(args.verbose) configurations = configuration_handler.read_configuration(args.config_input) if configurations: sys.exit(run_action(args, configurations, args.config_input)) diff --git a/bibigrid2/core/utility/ansible_commands.py b/bibigrid/core/utility/ansible_commands.py similarity index 93% rename from bibigrid2/core/utility/ansible_commands.py rename to bibigrid/core/utility/ansible_commands.py index c84030d87..3fdc7a940 100644 --- a/bibigrid2/core/utility/ansible_commands.py +++ b/bibigrid/core/utility/ansible_commands.py @@ -3,7 +3,7 @@ """ import os -import bibigrid2.core.utility.paths.ansible_resources_path as aRP +import bibigrid.core.utility.paths.ansible_resources_path as aRP #TO_LOG = "| sudo tee -a /var/log/ansible.log" #AIY = "apt-get -y install" @@ -45,7 +45,8 @@ # Execute PLAYBOOK_HOME = ("sudo mkdir -p /opt/playbook", "Create playbook home.") -PLAYBOOK_HOME_RIGHTS = ("sudo chown ubuntu:ubuntu /opt/playbook", "Adjust playbook home permission.") +PLAYBOOK_HOME_RIGHTS = ("uid=$(id -u); gid=$(id -g); sudo chown ${uid}:${gid} /opt/playbook", + "Adjust playbook home permission.") MV_ANSIBLE_CONFIG = ( "sudo install -D /opt/playbook/ansible.cfg /etc/ansible/ansible.cfg", "Move ansible configuration.") EXECUTE = (f"ansible-playbook {os.path.join(aRP.PLAYBOOK_PATH_REMOTE, aRP.SITE_YML)} -i " diff --git a/bibigrid2/core/utility/ansible_configurator.py b/bibigrid/core/utility/ansible_configurator.py similarity index 97% rename from bibigrid2/core/utility/ansible_configurator.py rename to bibigrid/core/utility/ansible_configurator.py index 0bce57c1e..a853fbcd2 100644 --- a/bibigrid2/core/utility/ansible_configurator.py +++ b/bibigrid/core/utility/ansible_configurator.py @@ -7,13 +7,13 @@ import mergedeep import yaml -from bibigrid2.core.actions import create -from bibigrid2.core.actions import ide -from bibigrid2.core.actions import list_clusters -from bibigrid2.core.utility.handler import configuration_handler -from bibigrid2.core.utility import id_generation -from bibigrid2.core.utility.paths import ansible_resources_path as aRP -from bibigrid2.core.utility import yaml_dumper +from bibigrid.core.actions import create +from bibigrid.core.actions import ide +from bibigrid.core.actions import list_clusters +from bibigrid.core.utility.handler import configuration_handler +from bibigrid.core.utility import id_generation +from bibigrid.core.utility.paths import ansible_resources_path as aRP +from bibigrid.core.utility import yaml_dumper DEFAULT_NFS_SHARES = ["/vol/spool"] ADDITIONAL_PATH = "additional/" diff --git a/bibigrid2/core/utility/command_line_interpreter.py b/bibigrid/core/utility/command_line_interpreter.py similarity index 76% rename from bibigrid2/core/utility/command_line_interpreter.py rename to bibigrid/core/utility/command_line_interpreter.py index b057bb82b..31d07e0b4 100644 --- a/bibigrid2/core/utility/command_line_interpreter.py +++ b/bibigrid/core/utility/command_line_interpreter.py @@ -3,10 +3,25 @@ """ import argparse +import logging import os STANDARD_CONFIG_INPUT_PATH = os.path.expanduser("~/.config/bibigrid") FOLDER_START = ("~/", "/") +LOG = logging.getLogger("bibigrid") + + +def check_cid(cid): + if "-" in cid: + new_cid = cid.split("-")[-1] + LOG.info("-cid %s is not a cid, but probably the entire master name. Using '%s' as " + "cid instead.", cid, new_cid) + return new_cid + if "." in cid: + LOG.info("-cid %s is not a cid, but probably the master's ip. " + "Using the master ip instead of cid only works if a cluster key is in your systems default ssh key " + "location (~/.ssh/). Otherwise bibigrid can't identify the cluster key.") + return cid def interpret_command_line(): @@ -14,7 +29,7 @@ def interpret_command_line(): Interprets commandline. Used in startup.py :return: """ - parser = argparse.ArgumentParser(description='Bibigrid2 sets up cluster easily inside a cloud environment') + parser = argparse.ArgumentParser(description='Bibigrid sets up cluster easily inside a cloud environment') parser.add_argument("-v", "--verbose", action="count", default=0, help="Increases logging verbosity. `-v` adds more info to the logfile, " "`-vv` adds debug information to the logfile.") @@ -24,7 +39,7 @@ def interpret_command_line(): "Relative paths can be used and start " "at ~/.config/bibigrid", required=True, type=lambda s: s if s.startswith(FOLDER_START) else os.path.join(STANDARD_CONFIG_INPUT_PATH, s)) - parser.add_argument("-cid", "--cluster_id", metavar="", type=str, default="", + parser.add_argument("-cid", "--cluster_id", metavar="", type=check_cid, default="", help="Cluster id is needed for ide and termination") actions = parser.add_mutually_exclusive_group(required=True) diff --git a/bibigrid2/core/utility/handler/cluster_ssh_handler.py b/bibigrid/core/utility/handler/cluster_ssh_handler.py similarity index 96% rename from bibigrid2/core/utility/handler/cluster_ssh_handler.py rename to bibigrid/core/utility/handler/cluster_ssh_handler.py index 78500ade0..138011d23 100644 --- a/bibigrid2/core/utility/handler/cluster_ssh_handler.py +++ b/bibigrid/core/utility/handler/cluster_ssh_handler.py @@ -5,7 +5,7 @@ import logging import os -from bibigrid2.core.actions import create, list_clusters +from bibigrid.core.actions import create, list_clusters LOG = logging.getLogger("bibigrid") def get_ssh_connection_info(cluster_id, master_provider, master_configuration): diff --git a/bibigrid2/core/utility/handler/configuration_handler.py b/bibigrid/core/utility/handler/configuration_handler.py similarity index 100% rename from bibigrid2/core/utility/handler/configuration_handler.py rename to bibigrid/core/utility/handler/configuration_handler.py diff --git a/bibigrid2/core/utility/handler/logging_path_handler.py b/bibigrid/core/utility/handler/logging_path_handler.py similarity index 100% rename from bibigrid2/core/utility/handler/logging_path_handler.py rename to bibigrid/core/utility/handler/logging_path_handler.py diff --git a/bibigrid2/core/utility/handler/provider_handler.py b/bibigrid/core/utility/handler/provider_handler.py similarity index 95% rename from bibigrid2/core/utility/handler/provider_handler.py rename to bibigrid/core/utility/handler/provider_handler.py index 45434505e..18668b2f9 100644 --- a/bibigrid2/core/utility/handler/provider_handler.py +++ b/bibigrid/core/utility/handler/provider_handler.py @@ -4,8 +4,8 @@ import logging -from bibigrid2.core.utility.handler import configuration_handler -from bibigrid2.openstack import openstack_provider +from bibigrid.core.utility.handler import configuration_handler +from bibigrid.openstack import openstack_provider PROVIDER_NAME_DICT = {"openstack": openstack_provider.OpenstackProvider} PROVIDER_CLASS_DICT = {provider.__name__: provider for provider in PROVIDER_NAME_DICT.values()} diff --git a/bibigrid2/core/utility/handler/ssh_handler.py b/bibigrid/core/utility/handler/ssh_handler.py similarity index 98% rename from bibigrid2/core/utility/handler/ssh_handler.py rename to bibigrid/core/utility/handler/ssh_handler.py index c0c6f152d..c8b44b8ac 100644 --- a/bibigrid2/core/utility/handler/ssh_handler.py +++ b/bibigrid/core/utility/handler/ssh_handler.py @@ -10,8 +10,8 @@ import paramiko import yaml -from bibigrid2.models.exceptions import ConnectionException, ExecutionException -from bibigrid2.core.utility import ansible_commands as aC +from bibigrid.models.exceptions import ConnectionException, ExecutionException +from bibigrid.core.utility import ansible_commands as aC PRIVATE_KEY_FILE = ".ssh/id_ecdsa" # to name bibigrid-temp keys identically on remote ANSIBLE_SETUP = [aC.NO_UPDATE, aC.UPDATE, diff --git a/bibigrid2/core/utility/id_generation.py b/bibigrid/core/utility/id_generation.py similarity index 97% rename from bibigrid2/core/utility/id_generation.py rename to bibigrid/core/utility/id_generation.py index 24d6718e5..8186d340a 100644 --- a/bibigrid2/core/utility/id_generation.py +++ b/bibigrid/core/utility/id_generation.py @@ -4,7 +4,7 @@ import shortuuid -from bibigrid2.core.actions import create +from bibigrid.core.actions import create MAX_ID_LENGTH = 15 CLUSTER_UUID_ALPHABET = '0123456789abcdefghijkmnopqrstuvwxyz' diff --git a/bibigrid2/core/utility/paths/ansible_resources_path.py b/bibigrid/core/utility/paths/ansible_resources_path.py similarity index 97% rename from bibigrid2/core/utility/paths/ansible_resources_path.py rename to bibigrid/core/utility/paths/ansible_resources_path.py index d48336568..070d287dd 100644 --- a/bibigrid2/core/utility/paths/ansible_resources_path.py +++ b/bibigrid/core/utility/paths/ansible_resources_path.py @@ -4,7 +4,7 @@ import os -import bibigrid2.core.utility.paths.basic_path as bP +import bibigrid.core.utility.paths.basic_path as bP # UNIVERSAL ANSIBLE_HOSTS: str = "ansible_hosts" diff --git a/bibigrid2/core/utility/paths/basic_path.py b/bibigrid/core/utility/paths/basic_path.py similarity index 100% rename from bibigrid2/core/utility/paths/basic_path.py rename to bibigrid/core/utility/paths/basic_path.py diff --git a/bibigrid2/core/utility/paths/bin_path.py b/bibigrid/core/utility/paths/bin_path.py similarity index 75% rename from bibigrid2/core/utility/paths/bin_path.py rename to bibigrid/core/utility/paths/bin_path.py index a99058dcb..16b10011d 100644 --- a/bibigrid2/core/utility/paths/bin_path.py +++ b/bibigrid/core/utility/paths/bin_path.py @@ -5,7 +5,7 @@ import os -import bibigrid2.core.utility.paths.basic_path as bP +import bibigrid.core.utility.paths.basic_path as bP BIN: str = "bin/" BIN_PATH: str = os.path.join(bP.RESOURCES_PATH, BIN) diff --git a/bibigrid2/core/utility/validate_configuration.py b/bibigrid/core/utility/validate_configuration.py similarity index 93% rename from bibigrid2/core/utility/validate_configuration.py rename to bibigrid/core/utility/validate_configuration.py index 93662d1fe..f69162396 100644 --- a/bibigrid2/core/utility/validate_configuration.py +++ b/bibigrid/core/utility/validate_configuration.py @@ -5,11 +5,12 @@ import logging import os -from bibigrid2.core.utility.handler import configuration_handler +from bibigrid.core.utility.handler import configuration_handler ACCEPTED_KEY_IDENTIFIERS = {"RSA": 4096, "ECDSA": 521, "ED25519": 256} LOG = logging.getLogger("bibigrid") + def evaluate(check_name, check_result): """ Logs check_resul as warning if failed and as success if succeeded. @@ -43,7 +44,7 @@ def check_provider_data(provider_data_list, provider_count): seen.append(elem) if duplicates: LOG.warning("Duplicate provider(s) %s. For each provider you can only create one configuration. " - "Please check your configurations.", duplicates) + "Please check your configurations.", duplicates) success = False else: LOG.info("All providers are unique.") @@ -69,14 +70,14 @@ def evaluate_ssh_public_key_file_security(ssh_public_key_file): minimum_size = ACCEPTED_KEY_IDENTIFIERS.get(identifier_clean) if not minimum_size: - LOG.warning("sshPublicKey '%s' is %s. Which secure length is unknown to bibigrid2.\n" - "Known encryptions are (with minimum size): %s", - ssh_public_key_file, identifier_clean, ACCEPTED_KEY_IDENTIFIERS) + LOG.warning("sshPublicKey '%s' is %s. Which secure length is unknown to bibigrid.\n" + "Known encryptions are (with minimum size): %s", + ssh_public_key_file, identifier_clean, ACCEPTED_KEY_IDENTIFIERS) else: LOG.info("sshPublicKey '%s' is a known encryption.", ssh_public_key_file) if minimum_size > int(length): LOG.warning("sshPublicKey '%s' is not long enough! %s should be >= %s, but is %s", - ssh_public_key_file, identifier_clean, minimum_size, int(length)) + ssh_public_key_file, identifier_clean, minimum_size, int(length)) else: LOG.info("sshPublicKey '%s' is long enough (%s/%s)!", ssh_public_key_file, int(length), minimum_size) return success @@ -109,12 +110,12 @@ def check_clouds_yaml_security(): """ success = True LOG.info("Checking validity of entire clouds.yaml and clouds-public.yaml") - clouds, clouds_public = configuration_handler.get_clouds_files() # pylint: disable=unused-variable + clouds, clouds_public = configuration_handler.get_clouds_files() # pylint: disable=unused-variable if clouds_public: for cloud in clouds_public: if clouds_public[cloud].get("profile"): LOG.warning(f"{cloud}: Profiles should be placed in clouds.yaml not clouds-public.yaml! " - f"Key ignored.") + f"Key ignored.") success = False if clouds_public[cloud].get("auth"): for key in ["password", "username", "application_credential_id", "application_credential_secret"]: @@ -140,8 +141,8 @@ def check_cloud_yaml(cloud_specification): and not ("auth_type" in keys and "application_credential_id" in auth_keys and "application_credential_secret" in auth_keys): LOG.warning("Insufficient authentication information. Needs either password and username or " - "if using application credentials: " - "auth_type, application_credential_id and application_credential_secret.") + "if using application credentials: " + "auth_type, application_credential_id and application_credential_secret.") success = False if "auth_url" not in auth_keys: LOG.warning("Authentification URL auth_url is missing.") @@ -200,7 +201,8 @@ def validate(self): checks = [("master/vpn", self.check_master_vpn_worker), ("servergroup", self.check_server_group), ("instances", self.check_instances), ("volumes", self.check_volumes), ("network", self.check_network), ("quotas", self.check_quotas), - ("sshPublicKeyFiles", self.check_ssh_public_key_files), ("cloudYamls", self.check_clouds_yamls)] + ("sshPublicKeyFiles", self.check_ssh_public_key_files), ("cloudYamls", self.check_clouds_yamls), + ("nfs", self.check_nfs)] if success: for check_name, check_function in checks: success = evaluate(check_name, check_function()) and success @@ -234,7 +236,7 @@ def check_provider_connections(self): providers_unconnectable.append(provider.name) if providers_unconnectable: LOG.warning("API connection to %s not successful. Please check your configuration.", - providers_unconnectable) + providers_unconnectable) success = False return success @@ -280,7 +282,7 @@ def check_instance(self, instance_name, instance, provider): if instance_image["status"] != "active": LOG.warning("Instance %s image: %s not active", instance_name, instance_image_id_or_name) print("Available active images:") - print("\n".join(provider.get_active_images)) + print("\n".join(provider.get_active_images())) return False LOG.info("Instance %s image: %s found", instance_name, instance_image_id_or_name) instance_type = instance["type"] @@ -437,3 +439,20 @@ def check_clouds_yamls(self): LOG.warning("Cloud specification %s is faulty. BiBiGrid understood %s.", index, cloud_specification) success = check_clouds_yaml_security() and success return success + + def check_nfs(self): + """ + Checks whether nfsshares => nfs holds and logs if failed. Returns True in every case as it is not fatale. + @return: True + """ + LOG.info("Checking nfs...") + success = True + master_configuration = self.configurations[0] + nfs_shares = master_configuration.get("nfsShares") + nfs = master_configuration.get("nfs") + if nfs_shares and not nfs: + success = True + LOG.warning("nfsShares exist, but nfs is False. nfsShares will be ignored!") + else: + success = True + return success diff --git a/bibigrid2/core/utility/yaml_dumper.py b/bibigrid/core/utility/yaml_dumper.py similarity index 100% rename from bibigrid2/core/utility/yaml_dumper.py rename to bibigrid/core/utility/yaml_dumper.py diff --git a/bibigrid2/models/exceptions.py b/bibigrid/models/exceptions.py similarity index 100% rename from bibigrid2/models/exceptions.py rename to bibigrid/models/exceptions.py diff --git a/bibigrid2/models/return_threading.py b/bibigrid/models/return_threading.py similarity index 100% rename from bibigrid2/models/return_threading.py rename to bibigrid/models/return_threading.py diff --git a/bibigrid2/openstack/openstack_provider.py b/bibigrid/openstack/openstack_provider.py similarity index 98% rename from bibigrid2/openstack/openstack_provider.py rename to bibigrid/openstack/openstack_provider.py index c41fce210..c27f13c8b 100644 --- a/bibigrid2/openstack/openstack_provider.py +++ b/bibigrid/openstack/openstack_provider.py @@ -11,10 +11,10 @@ from keystoneauth1.exceptions.http import NotFound from keystoneauth1.identity import v3 -from bibigrid2.core import provider -from bibigrid2.core.actions import create -from bibigrid2.core.actions import version -from bibigrid2.models.exceptions import ExecutionException +from bibigrid.core import provider +from bibigrid.core.actions import create +from bibigrid.core.actions import version +from bibigrid.models.exceptions import ExecutionException LOG = logging.getLogger("bibigrid") diff --git a/documentation/markdown/bibigrid_software_list.md b/documentation/markdown/bibigrid_software_list.md index d3c2d4d13..aca6a829e 100644 --- a/documentation/markdown/bibigrid_software_list.md +++ b/documentation/markdown/bibigrid_software_list.md @@ -1,8 +1,9 @@ # BiBiGrid Used Software -| Name | Purpose | Official Link | -|:------------------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------:| -| [Ansible](software/ansible.md) | Ansible, an open source community project by Red Hat, enables the idempotent setup of servers. Ansible is used to **prepare** all cluster nodes. | [Getting started with Ansible](https://docs.ansible.com/ansible/latest/getting_started/index.html) | -| [Slurm](software/slurm.md) | Slurm is an open source cluster management and job scheduling system. Slurm is used to **schedule** cluster nodes i.e. Slurm will start and shutdown nodes as needed. | [Quick Start User Guide](https://slurm.schedmd.com/quickstart.html) | -| [Theia IDE](software/theia_ide.md) | Theia IDE is a Web IDE, build using the Theia Framework, that allows easy, intuitive and abstract **web access** to cluster nodes. Theia IDE is optional. | [Using "Theia" as an End User](https://theia-ide.org/docs/user_getting_started/) | -| [Zabbix](software/zabbix.md) | Zabbix is an open source **monitoring** solution for networks, servers, clouds, applications and services. Zabbix is optional. | [What is Zabbix](https://www.zabbix.com/documentation/current/en/manual/introduction/about) | \ No newline at end of file +| Name | Purpose | Official Link | +|:----------------------------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------:|:--------------------------------------------------------------------------------------------------:| +| [Ansible](software/ansible.md) | Ansible, an open source community project by Red Hat, enables the idempotent setup of servers. Ansible is used to **prepare** all cluster nodes. | [Getting started with Ansible](https://docs.ansible.com/ansible/latest/getting_started/index.html) | +| [Slurm](software/slurm.md) | Slurm is an open source cluster management and job scheduling system. Slurm is used to **schedule** cluster nodes i.e. Slurm will start and shutdown nodes as needed. | [Quick Start User Guide](https://slurm.schedmd.com/quickstart.html) | +| [Theia IDE](software/theia_ide.md) | Theia IDE is a Web IDE, build using the Theia Framework, that allows easy, intuitive and abstract **web access** to cluster nodes. Theia IDE is optional. | [Using "Theia" as an End User](https://theia-ide.org/docs/user_getting_started/) | +| [Zabbix](software/zabbix.md) | Zabbix is an open source **monitoring** solution for networks, servers, clouds, applications and services. Zabbix is optional. | [What is Zabbix](https://www.zabbix.com/documentation/current/en/manual/introduction/about) | +| [NFS](software/nfs.md) | Network File System allows file access over a network similarly to local storage access. | [Getting started with NFS](https://www.redhat.com/sysadmin/getting-started-nfs) | diff --git a/documentation/markdown/features/cloud_specification_data.md b/documentation/markdown/features/cloud_specification_data.md index c70a776db..5230b6a83 100644 --- a/documentation/markdown/features/cloud_specification_data.md +++ b/documentation/markdown/features/cloud_specification_data.md @@ -1,6 +1,6 @@ # Cloud Specification Data -To access the cloud, authentication information is required. The BiBiGrid2 no longer uses environment variables, but a two file system instead. -`clouds.yaml` and `clouds-public.yaml` can be placed in `~/.config/bibigrid/` or `/etc/bibigrid/` and will be loaded by BiBiGrid2 on execution. +To access the cloud, authentication information is required. The BiBiGrid no longer uses environment variables, but a two file system instead. +`clouds.yaml` and `clouds-public.yaml` can be placed in `~/.config/bibigrid/` or `/etc/bibigrid/` and will be loaded by BiBiGrid on execution. While you store your password and username in `clouds.yaml` (private), you can store all other information ready to share in `clouds-public.yaml` (shareable). However, all information can just be stored in `clouds.yaml`. diff --git a/documentation/markdown/features/configuration.md b/documentation/markdown/features/configuration.md index 3e0309323..99fccbeec 100644 --- a/documentation/markdown/features/configuration.md +++ b/documentation/markdown/features/configuration.md @@ -30,10 +30,12 @@ created clusters to the owners of the private keyfile. For example, you can add him to access your started cluster later on to debug it. #### masterMounts (optional) -`masterMounts` expects a list of volumes or snapshots that will then be mounted to the master. If any snapshots are +`masterMounts` expects a list of volumes or snapshots that will then be attached to the master. If any snapshots are given, the related volumes are first created and then those volumes are used by BiBiGrid. Those volumes are not deleted after Cluster termination. +[[Link to mounting infomation]] #ToDo +
What is mounting? @@ -47,7 +49,7 @@ after Cluster termination. #### nfsShares (optional) `nfsShares` expects a list of folder paths to share using nfs. In every case, `/vol/spool/` is always an nfsShare. -This key only makes sense if the [nfs key](#nfs) is set `True`. +This key is only relevant if the [nfs key](#nfs--optional-) is set `True`.
@@ -140,6 +142,8 @@ Find your active `images`: openstack image list --os-cloud=openstack | grep active ``` +Currently, images based on Ubuntu 20.04/22.04 (Focal/Jammy) and Debian 11(Bullseye) are supported. + Find your active `flavors`: ``` diff --git a/documentation/markdown/software/nfs.md b/documentation/markdown/software/nfs.md new file mode 100644 index 000000000..9e47c349f --- /dev/null +++ b/documentation/markdown/software/nfs.md @@ -0,0 +1,74 @@ +# Network File System (NFS) +NFS is used as an abstraction layer to allow users to work naturally across file systems. +In a cluster setup working across file systems is really important when multiple nodes work on the same data. + +Most BiBiGrid users will never really interact consciously with BiBiGrid's underlying NFS, but simply use it. + +## How To Configure NFS Shares? +When starting an ansible cluster, at least `/vol/spool` is initialised as an NFS share if the key +[nfs](../features/configuration.md#nfs--optional-) is `True`. +Further NFS shares can then be configured using configuration's +[nfsshares](../features/configuration.md#nfsshares--optional-) key. +### Manually Creating NFS Shares +We discourage bypassing BiBiGrid's [configuration](../features/configuration.md#nfsshares--optional-) by creating +additional NFS shares manually, because they will not be automatically registered by scheduled workers. + +## Useful Commands + +| Summary | Command | Explanation & Comment | +|:----------------------:|:----------------------:|:------------------------------------------------------:| +| List all nfs shares | `showmount --exports` | | + +### NFS commands +See [nfs' manpage](https://man7.org/linux/man-pages/man5/nfs.5.html). + +## How To Share an Attached Volume +By mounting a volume into a shared directory, volumes can be shared. + +### Configuration +Let's assume our configuration holds (among others) the keys: +```yml +nfs: True +masterMounts: + - testMount + +nfsShares: + - testShare +``` + +Where `testMount` is an existing, formatted volume with a filesystem type (for example: ext4, ext3, ntfs, ...). + +During cluster creation... +1. BiBiGrid sets up the nfsShare `/testShare`. +2. BiBiGrid attached the volume `testMount` to our master instance. The volume is not mounted yet. + +3. We call the cluster `bibigrid-master-ournfsclusterid` in the following. + +### Mounting a Volume Into a Shared Directory +In order to mount a volume into a shared directory, we first need to identify where our volume was attached. + +#### Find Where Volume was attached +Executing this openstack client command will give us a list of volumes. +Most likely it is best run from your local machine. +```sh +openstack volume list --os-cloud=openstack +``` +Result: + +| ID | Name | Status | Size | Attached to | +|:------------------------------------:|--------------:|:---------:|:----:|:-------------------------------------------------------:| +| 42424242-4242-4242-4242-424242424242 | notTestVolume | available | X | | +| 42424242-4242-4242-4242-424242424221 | testMount | in-use | Y | Attached to bibigrid-master-ournfsclusterid on /dev/vdd | + +As you can see, the volume `testMount` was attached to `/dev/vdd`. +We can double-check whether `/dev/vdd` really exists by executing `lsblk` or `lsblk | grep /dev/vdd` on the master. + +#### Mount Volume into Share +As our NFS share is `/testShare`, we now need to mount `dev/vdd` into `testShare`: + +```sh +sudo mount -t auto /dev/vdd /testShare +``` + +The volume `testMount` is now successfully shared using NFS. +Workers can access the volume now by using `/testShare`, too. diff --git a/documentation/markdown/software/slurm.md b/documentation/markdown/software/slurm.md index a97a2b3f6..ac1bd1658 100644 --- a/documentation/markdown/software/slurm.md +++ b/documentation/markdown/software/slurm.md @@ -19,3 +19,120 @@ For more options see [slurm client's manpage](https://manpages.debian.org/testin |:---------------------------------------------------------------------------------:|:--------------------------------------------:| | [NODE STATE CODES](https://slurm.schedmd.com/sinfo.html#SECTION_NODE-STATE-CODES) | Very helpful to interpret `sinfo` correctly. | + +## REST API + +BiBiGrids configures Slurm's REST API Daemon listening on `0.0.0.0:6420`. + +Get token for user slurm +```shell +$ scontrol token -u slurm +SLURM_JWT=eyJhbGc... +``` + +Get openapi specification +```shell +$ token=eyJhbGc... +$ user=slurm +$ curl -H "X-SLURM-USER-NAME: $user" -H "X-SLURM-USER-TOKEN: $token" localhost:6820/openapi +... +``` + +### Read more + +| Summary | Explanation | +|:--------------------------------------------------------------------------:|:------------------------------------:| +| [Slurm REST API](https://slurm.schedmd.com/rest.html) | Slurm REST API documentation | +| [JSON Web Tokens (JWT) Authentication](https://slurm.schedmd.com/jwt.html) | Helpful to understand JWT/SlurmRestD | + + +## Slurm Packages + +You may have noticed that BiBiGrid doesn't use Slurm packages provided by the supported operating systems. +To be independent of the distributions release cycle we decided to build Slurm by ourselves. For those +who want to run a specific Slurm version the following documentation might be helpful. + +### Prepare build system + +At time of writing Slurm, 22.05.7 was the latest version available. Debian 11, Ubuntu 20.04/22.04 +as build system were successfully tested. + +``` +$ apt install tmux git build-essential vim curl + +$ mkdir build +$ cd build + +$ curl https://download.schedmd.com/slurm/slurm-22.05.7.tar.bz2 --output slurm-22.05.7.tar.bz2 +$ tar -xjf slurm-22.05.7.tar.bz2 +``` + +### Install build dependencies + +To enable source code repositories uncomment the lines starting with deb-src running : +```shell +$ sed -i.bak 's/^# *deb-src/deb-src/g' /etc/apt/sources.list && \ +apt-get update +``` + +With source repositories enabled install build dependencies of slurm-wlm. + +```shell +$ apt build-dep -y slurm-wlm +``` + +and for Ubuntu 20.04 only additionally + +```shell +$ apt install libyaml-dev libjson-c-dev libhttp-parser-dev libjwt-dev +``` + +To make use of [Control Group v2](https://slurm.schedmd.com/cgroup_v2.html) the development +files from dBus API must be installed additionally. + +```shell +apt install libdbus-1-dev +``` + +### Build slurm + +Building slurm is now an easy job. + +```shell +$ ./configure --prefix=/usr --sysconfdir=/etc/slurm --with-systemdsystemunitdir=/lib/systemd/system + +$ make -j 8 +``` + + +### Create deb package + +#### Determine dependencies + +```shell +$ python3 get_deb_dependencies.py --checkinstall --substr slurm slurm-wlm +libc6 \(\>= 2.29\), libhwloc15 \(\>= 2.1.0+dfsg\), liblz4-1 \(\>= 0.0~r130\), ... +``` + +#### Run Checkinstall + +Checkinstall makes it easy to create a debian package: + +```shell +$ checkinstall --type=debian --install=no --fstrans=no \ +--pkgname=slurm-full \ +--pkgversion=22.05.7 \ +--pkgrelease=1 \ +--pkglicense=GPL \ +--maintainer=jkrueger@cebitec.uni-bielefeld.de \ +--replaces=slurm-wlm \ +--requires="libc6 \(\>= 2.29\), libhwloc15 \(\>= 2.1.0+dfsg\), liblz4-1 \(\>= 0.0~r130\), libnuma1 \(\>= 2.0.11\), libpam0g \(\>= 0.99.7.1\), zlib1g \(\>= 1:1.2.0\), libcurl4 \(\>= 7.16.2\), libfreeipmi17 \(\>= 1.4.4\), libhdf5-103, libipmimonitoring6 \(\>= 1.1.5\), liblua5.1-0, libmunge2 \(\>= 0.5.8\), libmysqlclient21 \(\>= 8.0.11\), librrd8 \(\>= 1.3.0\), adduser, ucf, munge, lsb-base \(\>= 3.2-12\), libncurses6 \(\>= 6\), libreadline8 \(\>= 6.0\), libtinfo6 \(\>= 6\)" \ +--replaces="slurm-client, slurm-wlm, slurm-wlm-basic-plugins, slurmctld, slurmd, slurmdbd, slurmrestd" \ +--conflicts="slurm-client, slurm-wlm, slurm-wlm-basic-plugins, slurmctld, slurmd, slurmdbd, slurmrestd" +``` + +### Read more + +| Summary | Explanation | +|:--------------------------------------------------------------------------------------------------:|:-------------------------------------------:| +| [Woongbin's blog](https://wbk.one/%2Farticle%2F42a272c3%2Fapt-get-build-dep-to-install-build-deps) | Exhausted example for building deb packages | \ No newline at end of file diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 000000000..ef9bc405b --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,2 @@ +ansible_lint==6.8.0 +pylint==2.14.5 diff --git a/requirements.txt b/requirements.txt index 2cbce46da..8aa11c8ce 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,58 +1,6 @@ -appdirs==1.4.4 -attrs==22.1.0 -autopage==0.5.1 -bcrypt==4.0.1 -certifi==2022.9.24 -cffi==1.15.1 -charset-normalizer==2.1.1 -cliff==4.1.0 -cmd2==2.4.2 -cryptography==38.0.4 -debtcollector==2.5.0 -decorator==5.1.1 -dogpile.cache==1.1.8 -idna==3.4 -importlib-metadata==5.1.0 -iso8601==1.1.0 -jmespath==1.0.1 -jsonpatch==1.32 -jsonpointer==2.3 -keystoneauth1==5.1.0 -mergedeep==1.3.4 -msgpack==1.0.4 -munch==2.5.0 -netaddr==0.8.0 -netifaces==0.11.0 -openstacksdk==0.103.0 -os-service-types==1.7.0 -osc-lib==2.6.2 -oslo.config==9.0.0 -oslo.i18n==5.1.0 -oslo.serialization==5.0.0 -oslo.utils==6.1.0 -packaging==21.3 -paramiko==2.12.0 -pbr==5.11.0 -prettytable==3.5.0 -pycparser==2.21 -PyNaCl==1.5.0 -pyparsing==3.0.9 -pyperclip==1.8.2 -python-cinderclient==9.1.0 -python-keystoneclient==5.0.1 -python-novaclient==18.2.0 -python-openstackclient==6.0.0 -pytz==2022.6 -PyYAML==6.0 -requests==2.28.1 -requestsexceptions==1.4.0 -rfc3986==2.0.0 -shortuuid==1.0.11 -simplejson==3.18.0 -six==1.16.0 -sshtunnel==0.4.0 -stevedore==4.1.1 -urllib3==1.26.13 -wcwidth==0.2.5 -wrapt==1.14.1 -zipp==3.11.0 +mergedeep +openstacksdk +paramiko +python-openstackclient +sshtunnel +shortuuid diff --git a/resources/playbook/roles/bibigrid/files/slurm/cgroup.conf b/resources/playbook/roles/bibigrid/files/slurm/cgroup.conf index 2b8ba9c6a..657a243d3 100644 --- a/resources/playbook/roles/bibigrid/files/slurm/cgroup.conf +++ b/resources/playbook/roles/bibigrid/files/slurm/cgroup.conf @@ -1,9 +1,6 @@ CgroupMountpoint="/sys/fs/cgroup" CgroupAutomount=yes -CgroupReleaseAgentDir="/etc/slurm/cgroup" -AllowedDevicesFile="/etc/slurm/cgroup_allowed_devices_file.conf" ConstrainCores=no -TaskAffinity=no ConstrainRAMSpace=yes ConstrainSwapSpace=no ConstrainDevices=no diff --git a/resources/playbook/roles/bibigrid/files/slurm/cgroup_allowed_devices_file.conf b/resources/playbook/roles/bibigrid/files/slurm/cgroup_allowed_devices_file.conf deleted file mode 100644 index 471ad8cfd..000000000 --- a/resources/playbook/roles/bibigrid/files/slurm/cgroup_allowed_devices_file.conf +++ /dev/null @@ -1,7 +0,0 @@ -/dev/null -/dev/urandom -/dev/zero -/dev/sd* -/dev/vd* -/dev/cpu/*/* -/dev/pts/* \ No newline at end of file diff --git a/resources/playbook/roles/bibigrid/files/slurm/slurmrestd_default b/resources/playbook/roles/bibigrid/files/slurm/default/slurmrestd similarity index 73% rename from resources/playbook/roles/bibigrid/files/slurm/slurmrestd_default rename to resources/playbook/roles/bibigrid/files/slurm/default/slurmrestd index b6d2fd860..50afbde7f 100644 --- a/resources/playbook/roles/bibigrid/files/slurm/slurmrestd_default +++ b/resources/playbook/roles/bibigrid/files/slurm/default/slurmrestd @@ -4,6 +4,6 @@ SLURM_CONF="/etc/slurm/slurm.conf" #SLURMRESTD_DEBUG="8" SLURM_JWT="" -SLURMRESTD_LISTEN=":6820" +SLURMRESTD_LISTEN="0.0.0.0:6820" SLURMRESTD_AUTH_TYPES="rest_auth/jwt" -SLURMRESTD_OPENAPI_PLUGINS="openapi/v0.0.36" \ No newline at end of file +SLURMRESTD_OPENAPI_PLUGINS="openapi/v0.0.38" \ No newline at end of file diff --git a/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmctld.override b/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmctld.override new file mode 100644 index 000000000..b897b2308 --- /dev/null +++ b/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmctld.override @@ -0,0 +1 @@ +# add your overrides here \ No newline at end of file diff --git a/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmd.override b/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmd.override new file mode 100644 index 000000000..9afe0f8b7 --- /dev/null +++ b/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmd.override @@ -0,0 +1,3 @@ +# Override systemd service to set conditional path +[Unit] +ConditionPathExists=/etc/slurm/slurm.conf \ No newline at end of file diff --git a/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmdbd.override b/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmdbd.override new file mode 100644 index 000000000..b897b2308 --- /dev/null +++ b/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmdbd.override @@ -0,0 +1 @@ +# add your overrides here \ No newline at end of file diff --git a/resources/playbook/roles/bibigrid/files/slurm/slurmrestd_override.conf b/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmrestd.override similarity index 68% rename from resources/playbook/roles/bibigrid/files/slurm/slurmrestd_override.conf rename to resources/playbook/roles/bibigrid/files/slurm/systemd/slurmrestd.override index eebbe66f7..90375b48c 100644 --- a/resources/playbook/roles/bibigrid/files/slurm/slurmrestd_override.conf +++ b/resources/playbook/roles/bibigrid/files/slurm/systemd/slurmrestd.override @@ -2,5 +2,9 @@ [Unit] After=slurmdbd.service [Service] +EnvironmentFile= +EnvironmentFile=/etc/default/slurmrestd +User=nobody +Group=nogroup ExecStart= ExecStart=/usr/sbin/slurmrestd $SLURMRESTD_OPTIONS \ No newline at end of file diff --git a/resources/playbook/roles/bibigrid/tasks/000-playbook-rights-server.yml b/resources/playbook/roles/bibigrid/tasks/000-playbook-rights-server.yml index b57c5a54c..438e17ac9 100644 --- a/resources/playbook/roles/bibigrid/tasks/000-playbook-rights-server.yml +++ b/resources/playbook/roles/bibigrid/tasks/000-playbook-rights-server.yml @@ -5,7 +5,6 @@ - name: Change mode of /opt/slurm directory file: - owner: ubuntu group: ansible path: /opt/playbook/ state: directory diff --git a/resources/playbook/roles/bibigrid/tasks/001-apt.yml b/resources/playbook/roles/bibigrid/tasks/001-apt.yml index 5d3b58f74..8a0243a7f 100644 --- a/resources/playbook/roles/bibigrid/tasks/001-apt.yml +++ b/resources/playbook/roles/bibigrid/tasks/001-apt.yml @@ -46,6 +46,26 @@ tags: - skip_ansible_lint +- name: Add apt.bi.denbi.de repository key + apt_key: + url: 'https://apt.bi.denbi.de/repo_key.key' + state: present + +- name: Add apt.bi.denbi.de repository + apt_repository: + repo: 'deb https://apt.bi.denbi.de/repos/apt/{{ ansible_distribution_release | lower }} {{ ansible_distribution_release | lower }} main' + +- when: ansible_distribution_release == 'focal' + block: + - name: Add mariadb.org repository key + apt_key: + url: 'https://mariadb.org/mariadb_release_signing_key.asc' + state: present + + - name: Add mariadb.org repository + apt_repository: + repo: 'deb https://mirror.netcologne.de/mariadb/repo/10.5/ubuntu focal main' + - name: Update apt cache apt: upgrade: 'yes' diff --git a/resources/playbook/roles/bibigrid/tasks/010-bin-server.yml b/resources/playbook/roles/bibigrid/tasks/010-bin-server.yml index f8635ef1f..446b7cc01 100644 --- a/resources/playbook/roles/bibigrid/tasks/010-bin-server.yml +++ b/resources/playbook/roles/bibigrid/tasks/010-bin-server.yml @@ -14,9 +14,7 @@ copy: src: ~/bin dest: /usr/local - owner: ubuntu - group: ubuntu - mode: '0774' + mode: '0775' - name: Delete origin folder file: path: ~{{ ansible_facts.env.SUDO_USER }}/bin diff --git a/resources/playbook/roles/bibigrid/tasks/011-zabbix-server.yml b/resources/playbook/roles/bibigrid/tasks/011-zabbix-server.yml index ea830b4d1..e9986be35 100644 --- a/resources/playbook/roles/bibigrid/tasks/011-zabbix-server.yml +++ b/resources/playbook/roles/bibigrid/tasks/011-zabbix-server.yml @@ -46,6 +46,8 @@ ignore_errors: true changed_when: false register: zabbix_schema_exists + tags: + - skip_ansible_lint - name: Import initial db schema shell: "set -o pipefail && zcat /usr/share/zabbix-sql-scripts/mysql/server.sql.gz | \ diff --git a/resources/playbook/roles/bibigrid/tasks/020-disk-worker.yml b/resources/playbook/roles/bibigrid/tasks/020-disk-worker.yml index 77f31b16f..4a352eaf3 100644 --- a/resources/playbook/roles/bibigrid/tasks/020-disk-worker.yml +++ b/resources/playbook/roles/bibigrid/tasks/020-disk-worker.yml @@ -1,14 +1,14 @@ - when: "'ephemeral' in group_names" block: - - name: Mount ephemeral - mount: - path: /vol/scratch - src: /mnt - fstype: none - opts: bind,auto - state: mounted - - name: Set 0777 rights for ephemeral mount - file: - path: /vol/scratch - state: directory - mode: 0777 + - name: Mount ephemeral + mount: + path: /vol/scratch + src: /mnt + fstype: none + opts: bind,auto + state: mounted + - name: Set 0777 rights for ephemeral mount + file: + path: /vol/scratch + state: directory + mode: 0777 diff --git a/resources/playbook/roles/bibigrid/tasks/042-slurm-server.yml b/resources/playbook/roles/bibigrid/tasks/042-slurm-server.yml index bae35efaf..ab460575b 100644 --- a/resources/playbook/roles/bibigrid/tasks/042-slurm-server.yml +++ b/resources/playbook/roles/bibigrid/tasks/042-slurm-server.yml @@ -12,12 +12,6 @@ state: present login_unix_socket: /var/run/mysqld/mysqld.sock -- name: Install Slurm database and RestAPI packages - apt: - name: - - slurmdbd - - slurmrestd - - name: Create slurmdb configuration file template: src: slurm/slurmdbd.conf @@ -43,31 +37,48 @@ - name: Copy env file for configuration of slurmrestd copy: - src: slurm/slurmrestd_default + src: slurm/default/slurmrestd dest: /etc/default/slurmrestd owner: root group: root mode: "0644" notify: - - slurmdbd - slurmrestd -- name: Create Service Directory +- name: Create system overrides directories (slurmdbdm slurmrestd) file: - path: /etc/systemd/system/slurmrestd.service.d + path: "/etc/systemd/system/{{ item }}.service.d" group: root owner: root mode: "0755" state: directory + with_items: + - slurmdbd + - slurmrestd -- name: Copy systemd Service override file +- name: Copy system overrides (slurmdbdm slurmrestd) copy: - src: slurm/slurmrestd_override.conf - dest: /etc/systemd/system/slurmrestd.service.d/override.conf + src: "slurm/systemd/{{ item }}.override" + dest: "/etc/systemd/system/{{ item }}.service.d/override.conf" mode: "0644" owner: root group: root + with_items: + - slurmdbd + - slurmrestd notify: + - slurmdbd + - slurmrestd + +- name: Enable slurmdbd and slurmrestd services + systemd: + name: "{{ item }}" + enabled: true + masked: false + state: started + daemon_reload: true + with_items: + - slurmdbd - slurmrestd - name: Start slurm explicit after all dependencies are configured @@ -82,11 +93,6 @@ args: executable: bash -# - name: Install python3-virtualenv -# apt: -# name: python3-virtualenv -# state: present - - name: Change mode of /opt/slurm directory file: owner: slurm @@ -95,17 +101,6 @@ state: directory mode: "0770" -# - name: Create slurm venv in opt -# command: -# cmd: virtualenv /opt/slurm/venv -p python3.10 -# creates: "/opt/slurm/venv" - -- name: Ensures /etc/slurm dir exists - file: - path: /etc/slurm/ - state: directory - mode: 0755 - - name: Ensures /opt/slurm/.ssh/ dir exists file: path: /opt/slurm/.ssh/ @@ -155,18 +150,6 @@ mode: "0750" -# - name: Copy requirements to correct location -# copy: -# src: slurm/requirements.txt -# dest: /opt/slurm/requirements.txt -# #owner: slurm -# group: ansible -# mode: "0550" - -# - name: Install requirements -# pip: requirements=/opt/slurm/requirements.txt -# virtualenv=/opt/slurm/venv - - name: Install python dependencies pip: name: diff --git a/resources/playbook/roles/bibigrid/tasks/042-slurm.yml b/resources/playbook/roles/bibigrid/tasks/042-slurm.yml index 5d6b3cf2c..3d2cb4aa2 100644 --- a/resources/playbook/roles/bibigrid/tasks/042-slurm.yml +++ b/resources/playbook/roles/bibigrid/tasks/042-slurm.yml @@ -1,7 +1,18 @@ -- name: Install Slurm base packages and dependencies +- name: Create Slurm group + group: + name: slurm + gid: 64030 + +- name: Create Slurm user + user: + name: slurm + uid: 64030 + group: slurm + +- name: Install Slurm package (and dependencies) apt: name: - - slurm-wlm + - slurm-full - munge - name: Create new secret (Munge) @@ -14,13 +25,55 @@ notify: - munge -- name: Disable slurmctld (on worker) +- name: Ensures slurm directories exists + file: + path: "{{ item }}" + state: directory + mode: 0775 + owner: root + group: slurm + with_items: + - /etc/slurm + - /var/lib/slurm + - /var/log/slurm + +- name: Create system overrides directories (slurmd, slurmctld) + file: + path: "/etc/systemd/system/{{ item }}.service.d" + group: root + owner: root + mode: "0755" + state: directory + with_items: + - slurmd + - slurmctld + +- name: Copy system overrides (slurmd, slurmctld) + copy: + src: "slurm/systemd/{{ item }}.override" + dest: "/etc/systemd/system/{{ item }}.service.d/override.conf" + mode: "0644" + owner: root + group: root + with_items: + - slurmd + - slurmctld + notify: + - slurmd + - slurmctld + +- name: Enable slurmctld and slurmd services systemd: - name: slurmctld - enabled: false - when: "'master' not in group_names" + name: "{{ item }}" + enabled: true + masked: false + state: started + daemon_reload: true + with_items: + - slurmctld + - slurmd -- name: SLURM configuration +- name: Create Slurm configuration template: src: slurm/slurm.conf dest: /etc/slurm/slurm.conf @@ -31,7 +84,7 @@ - slurmctld - slurmd -- name: SLURM cgroup configuration +- name: Slurm cgroup configuration copy: src: slurm/cgroup.conf dest: /etc/slurm/cgroup.conf @@ -41,14 +94,3 @@ notify: - slurmctld - slurmd - -- name: SLURM cgroup allowed devices conf - copy: - src: slurm/cgroup_allowed_devices_file.conf - dest: /etc/slurm/cgroup_allowed_devices_file.conf - owner: root - group: root - mode: 0444 - notify: - - slurmctld - - slurmd diff --git a/resources/playbook/roles/bibigrid/tasks/999-theia.yml b/resources/playbook/roles/bibigrid/tasks/999-theia.yml index 578c41294..d4c55f8da 100644 --- a/resources/playbook/roles/bibigrid/tasks/999-theia.yml +++ b/resources/playbook/roles/bibigrid/tasks/999-theia.yml @@ -44,6 +44,8 @@ creates: "{{ nvm_install_dir }}/nvm.sh" executable: bash warn: false + tags: + - skip_ansible_lint - name: Install node 16 and yarn shell: | diff --git a/resources/playbook/roles/bibigrid/templates/slurm/slurm.conf b/resources/playbook/roles/bibigrid/templates/slurm/slurm.conf index c448386aa..c8f97edf9 100644 --- a/resources/playbook/roles/bibigrid/templates/slurm/slurm.conf +++ b/resources/playbook/roles/bibigrid/templates/slurm/slurm.conf @@ -18,7 +18,7 @@ JobAcctGatherType=jobacct_gather/linux # see https://slurm.schedmd.com/slurm.conf.html#OPT_cloud_dns:~:text=for%20additional%20details.-,cloud_dns,-By%20default%2C%20Slurm # SlurmctldParameters=cloud_dns -# Funktioniert nicht wie vermutet. slurmctld versucht mit diesem Paramter schon beim Start alle Clients aufzulösen, +# Funktioniert nicht wie vermutet. slurmctld versucht mit diesem Parameter schon beim Start alle Clients aufzulösen, # was natürlich nicht funktioniert. # PRIORITY diff --git a/tests/provider/test_Provider.py b/tests/provider/test_Provider.py index 197a86d93..9575e16e5 100644 --- a/tests/provider/test_Provider.py +++ b/tests/provider/test_Provider.py @@ -1,9 +1,9 @@ import os import unittest -import bibigrid2.core.utility.handler.provider_handler as providerHandler -import bibigrid2.core.utility.handler.configuration_handler as configurationHandler -import bibigrid2.core.utility.paths.basic_path as bP +import bibigrid.core.utility.handler.configuration_handler as configurationHandler +import bibigrid.core.utility.handler.provider_handler as providerHandler +import bibigrid.core.utility.paths.basic_path as bP SERVER_KEYS = {'id', 'name', 'flavor', 'image', 'block_device_mapping', 'location', 'volumes', 'has_config_drive', 'host_id', 'progress', 'disk_config', 'power_state', 'task_state', @@ -62,7 +62,7 @@ PROVIDERS = providerHandler.get_providers(CONFIGURATIONS) -class ProviderServer(object): +class ProviderServer: def __init__(self, provider, name, configuration, key_name=None): self.provider = provider self.name = name @@ -73,7 +73,7 @@ def __init__(self, provider, name, configuration, key_name=None): def __enter__(self): return self.server_dict - def __exit__(self, type, value, traceback): + def __exit__(self, not_type, value, traceback): # type self.provider.delete_server(name_or_id=self.name) @@ -120,26 +120,26 @@ def test_server_start_attribute_error(self): def test_create_keypair_create_delete_false_delete(self): for provider in PROVIDERS: with self.subTest(provider.NAME): - provider.create_keypair("bibigrid2_test_keypair", KEYPAIR) - self.assertTrue(provider.delete_keypair("bibigrid2_test_keypair")) - self.assertFalse(provider.delete_keypair("bibigrid2_test_keypair")) + provider.create_keypair("bibigrid_test_keypair", KEYPAIR) + self.assertTrue(provider.delete_keypair("bibigrid_test_keypair")) + self.assertFalse(provider.delete_keypair("bibigrid_test_keypair")) def test_active_server_methods(self): for provider, configuration in zip(PROVIDERS, CONFIGURATIONS): - provider.create_keypair("bibigrid2_test_keypair", KEYPAIR) + provider.create_keypair("bibigrid_test_keypair", KEYPAIR) with self.subTest(provider.NAME): - with ProviderServer(provider, "bibigrid2_test_server", configuration, "bibigrid2_test_keypair") as ps: + with ProviderServer(provider, "bibigrid_test_server", configuration, "bibigrid_test_keypair") as ps: floating_ip = provider.create_floating_ip(provider.get_external_network(configuration["network"]), ps) server_list = provider.list_servers() self.assertEqual(SERVER_KEYS, set(ps.keys())) - self.assertEqual("bibigrid2_test_keypair", ps["key_name"]) + self.assertEqual("bibigrid_test_keypair", ps["key_name"]) self.assertEqual(FLOATING_IP_KEYS, set(floating_ip.keys())) - self.assertTrue([server for server in server_list if server["name"] == "bibigrid2_test_server" and + self.assertTrue([server for server in server_list if server["name"] == "bibigrid_test_server" and server["public_v4"] == floating_ip.floating_ip_address]) - provider.delete_keypair("bibigrid2_test_keypair") + provider.delete_keypair("bibigrid_test_keypair") def test_get_external_network(self): for provider, configuration in zip(PROVIDERS, CONFIGURATIONS): @@ -191,7 +191,7 @@ def test_get_flavor_detail(self): self.assertEqual(FLAVOR_KEYS, set(provider.get_flavor(configuration["flavor"]).keys())) def test_get_image(self): - for provider,configuration in zip(PROVIDERS, CONFIGURATIONS): + for provider, configuration in zip(PROVIDERS, CONFIGURATIONS): with self.subTest(provider.NAME): self.assertEqual(IMAGE_KEYS, set(provider.get_image_by_id_or_name(configuration["image"]).keys())) @@ -205,7 +205,8 @@ def test_get_snapshot(self): for provider, configuration in zip(PROVIDERS, CONFIGURATIONS): with self.subTest(provider.NAME): self.assertEqual(SNAPSHOT_KEYS, - set(provider.get_volume_snapshot_by_id_or_name(configuration["snapshot_image"]).keys())) + set(provider.get_volume_snapshot_by_id_or_name( + configuration["snapshot_image"]).keys())) def test_create_volume_from_snapshot(self): for provider, configuration in zip(PROVIDERS, CONFIGURATIONS): diff --git a/tests/test_ValidateConfiguration.py b/tests/test_ValidateConfiguration.py index 7bfd7d83a..96ffad267 100644 --- a/tests/test_ValidateConfiguration.py +++ b/tests/test_ValidateConfiguration.py @@ -2,14 +2,15 @@ from unittest import TestCase from unittest.mock import Mock, patch, MagicMock, call -import bibigrid2.core.utility.validate_configuration as validateConfiguration +from bibigrid.core.utility import validate_configuration class TestValidateConfiguration(TestCase): + # pylint: disable=R0904 def test_check_provider_data_count(self): provider_data_1 = {"PROJECT_ID": "abcd", "PROJECT_NAME": "1234"} provider_data_2 = {"PROJECT_ID": "9999", "PROJECT_NAME": "9999"} - vc = validateConfiguration + vc = validate_configuration self.assertTrue(vc.check_provider_data([provider_data_1, provider_data_2], 2)) self.assertFalse(vc.check_provider_data([provider_data_1, provider_data_2], 3)) self.assertTrue(vc.check_provider_data([], 0)) @@ -17,7 +18,7 @@ def test_check_provider_data_count(self): def test_check_provider_data_unique(self): provider_data_1 = {"PROJECT_ID": "abcd", "PROJECT_NAME": "1234"} provider_data_2 = {"PROJECT_ID": "9999", "PROJECT_NAME": "9999"} - vc = validateConfiguration + vc = validate_configuration self.assertTrue(vc.check_provider_data([provider_data_1, provider_data_2], 2)) self.assertFalse(vc.check_provider_data([provider_data_1, provider_data_1], 2)) self.assertTrue(vc.check_provider_data([], 0)) @@ -28,7 +29,7 @@ def test_check_master_vpn_worker_ordered(self): vpn_master = {} vpn_master.update(master) vpn_master.update(vpn) - vc = validateConfiguration.ValidateConfiguration(providers=None, configurations=[master]) + vc = validate_configuration.ValidateConfiguration(providers=None, configurations=[master]) self.assertTrue(vc.check_master_vpn_worker()) vc.configurations = [master, vpn] self.assertTrue(vc.check_master_vpn_worker()) @@ -43,71 +44,72 @@ def test_check_master_vpn_worker_unique(self): vpn_master = {} vpn_master.update(master) vpn_master.update(vpn) - vc = validateConfiguration.ValidateConfiguration(providers=None, configurations=[vpn_master]) + vc = validate_configuration.ValidateConfiguration(providers=None, configurations=[vpn_master]) self.assertFalse(vc.check_master_vpn_worker()) vc.configurations = [master, vpn_master] self.assertFalse(vc.check_master_vpn_worker()) def test_evaluate(self): - vc = validateConfiguration + vc = validate_configuration self.assertTrue(vc.evaluate("some", True)) self.assertFalse(vc.evaluate("some", False)) def test_check_provider_connection(self): mock = Mock() mock.conn = False - vc = validateConfiguration.ValidateConfiguration(providers=[mock], configurations=None) + vc = validate_configuration.ValidateConfiguration(providers=[mock], configurations=None) self.assertFalse(vc.check_provider_connections()) mock.conn = True self.assertTrue(vc.check_provider_connections()) def test_check_instances_master(self): - vc = validateConfiguration.ValidateConfiguration(providers=["31"], configurations=[{"masterInstance": "42"}]) + vc = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{"masterInstance": "42"}]) with patch.object(vc, "check_instance") as mock: vc.check_instances() mock.assert_called_with("masterInstance", "42", "31") def test_check_instances_vpn(self): - vc = validateConfiguration.ValidateConfiguration(providers=["31"], configurations=[{"vpnInstance": "42"}]) + vc = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{"vpnInstance": "42"}]) with patch.object(vc, "check_instance") as mock: vc.check_instances() mock.assert_called_with("vpnInstance", "42", "31") def test_check_instances_vpn_worker(self): - vc = validateConfiguration.ValidateConfiguration(providers=["31"], configurations=[ + vc = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[ {"masterInstance": "42", "workerInstances": ["42"]}]) with patch.object(vc, "check_instance") as mock: vc.check_instances() mock.assert_called_with("workerInstance", "42", "31") def test_check_instances_vpn_master_missing(self): - vc = validateConfiguration.ValidateConfiguration(providers=["31"], configurations=[{}]) + vc = validate_configuration.ValidateConfiguration(providers=["31"], configurations=[{}]) self.assertFalse(vc.check_instances()) - vc = validateConfiguration.ValidateConfiguration(providers=["31"], configurations=[{"workerInstances": ["42"]}]) + vc = validate_configuration.ValidateConfiguration(providers=["31"], + configurations=[{"workerInstances": ["42"]}]) self.assertFalse(vc.check_instances()) def test_check_instances_vpn_master_count(self): for i in range(3): - vc = validateConfiguration.ValidateConfiguration(providers=["31"] * i, - configurations=[{"masterInstance": "42"}] * i) - with patch.object(vc, "check_instance") as mock: - vc.check_instances() - self.assertTrue(vc.required_resources_dict["floating_ips"] == i) + vc = validate_configuration.ValidateConfiguration(providers=["31"] * i, + configurations=[{"masterInstance": "42"}] * i) + # with patch.object(vc, "check_instance") as mock: + vc.check_instances() + self.assertTrue(vc.required_resources_dict["floating_ips"] == i) def test_check_instance_image_not_found(self): - vc = validateConfiguration.ValidateConfiguration(providers=None, configurations=None) + vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) provider = Mock() provider.get_image_by_id_or_name = MagicMock(return_value=None) self.assertFalse(vc.check_instance(None, {"count": 1, "image": 2}, provider)) def test_check_instance_image_not_active(self): - vc = validateConfiguration.ValidateConfiguration(providers=None, configurations=None) + vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) provider = Mock() provider.get_image_by_id_or_name = MagicMock(return_value={"status": None}) self.assertFalse(vc.check_instance(None, {"count": 1, "image": 2}, provider)) def test_check_instance_image_active_combination_call(self): - vc = validateConfiguration.ValidateConfiguration(providers=None, configurations=None) + vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) provider = Mock() provider.get_image_by_id_or_name = MagicMock(return_value={"status": "active"}) with patch.object(vc, "check_instance_type_image_combination") as mock: @@ -118,26 +120,26 @@ def test_check_instance_image_not_found_count(self): provider = Mock() provider.get_image_by_id_or_name = MagicMock(return_value=None) for i in range(1, 3): - vc = validateConfiguration.ValidateConfiguration(providers=None, configurations=None) + vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) vc.check_instance(None, {"count": i, "image": 2}, provider) self.assertTrue(vc.required_resources_dict["instances"] == i) def test_check_instance_type_image_combination_has_enough_calls(self): - vc = validateConfiguration.ValidateConfiguration(providers=None, configurations=None) + vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) provider = MagicMock() provider.get_flavor.return_value = {"disk": 42, "ram": 32, "vcpus": 10} provider.get_image_by_id_or_name.return_value = {"minDisk": 22, "minRam": 12} with patch.object(vc, "has_enough") as mock: vc.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", provider=provider) - self.assertEquals(call(42, 22, "Type de.NBI tiny", "disk space"), mock.call_args_list[0]) - self.assertEquals(call(32, 12, "Type de.NBI tiny", "ram"), mock.call_args_list[1]) + self.assertEqual(call(42, 22, "Type de.NBI tiny", "disk space"), mock.call_args_list[0]) + self.assertEqual(call(32, 12, "Type de.NBI tiny", "ram"), mock.call_args_list[1]) def test_check_instance_type_image_combination_result(self): provider = MagicMock() provider.get_flavor.return_value = {"disk": 42, "ram": 32, "vcpus": 10} provider.get_image_by_id_or_name.return_value = {"minDisk": 22, "minRam": 12} - vc = validateConfiguration.ValidateConfiguration(providers=None, configurations=None) + vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) with patch.object(vc, "has_enough") as mock: mock.side_effect = [True, True, False, False, True, False, False, True] # True True @@ -158,32 +160,32 @@ def test_check_instance_type_image_combination_count(self): provider = MagicMock() provider.get_flavor.return_value = {"disk": 42, "ram": i * 32, "vcpus": i * 10} provider.get_image_by_id_or_name.return_value = {"minDisk": 22, "minRam": 12} - vc = validateConfiguration.ValidateConfiguration(providers=None, configurations=None) + vc = validate_configuration.ValidateConfiguration(providers=None, configurations=None) with patch.object(vc, "has_enough") as mock: vc.check_instance_type_image_combination(instance_image=None, instance_type="de.NBI tiny", provider=provider) - self.assertEquals(32 * i, vc.required_resources_dict["total_ram"]) - self.assertEquals(10 * i, vc.required_resources_dict["total_cores"]) + self.assertEqual(32 * i, vc.required_resources_dict["total_ram"]) + self.assertEqual(10 * i, vc.required_resources_dict["total_cores"]) mock.assert_called_with(32 * i, 12, 'Type de.NBI tiny', 'ram') def test_check_volumes_none(self): - vc = validateConfiguration.ValidateConfiguration(providers=[42], configurations=[{}]) + vc = validate_configuration.ValidateConfiguration(providers=[42], configurations=[{}]) self.assertTrue(vc.check_volumes()) def test_check_volumes_mismatch(self): provider = Mock() provider.get_volume_by_id_or_name = MagicMock(return_value=None) provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value=None) - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{"masterMounts": ["Test"]}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"masterMounts": ["Test"]}]) self.assertFalse(vc.check_volumes()) def test_check_volumes_match_snapshot(self): provider = Mock() provider.get_volume_by_id_or_name = MagicMock(return_value=None) provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value={"size": 1}) - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{"masterMounts": ["Test"]}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"masterMounts": ["Test"]}]) self.assertTrue(vc.check_volumes()) def test_check_volumes_match_snapshot_count(self): @@ -191,8 +193,8 @@ def test_check_volumes_match_snapshot_count(self): provider = Mock() provider.get_volume_by_id_or_name = MagicMock(return_value=None) provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value={"size": i}) - vc = validateConfiguration.ValidateConfiguration(providers=[provider] * i, - configurations=[{"masterMounts": ["Test"] * i}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider] * i, + configurations=[{"masterMounts": ["Test"] * i}]) self.assertTrue(vc.check_volumes()) self.assertTrue(vc.required_resources_dict["Volumes"] == i) self.assertTrue(vc.required_resources_dict["VolumeGigabytes"] == i ** 2) @@ -201,8 +203,8 @@ def test_check_volumes_match_volume(self): provider = Mock() provider.get_volume_by_id_or_name = MagicMock(return_value={"size": 1}) provider.get_volume_snapshot_by_id_or_name = MagicMock(return_value=None) - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{"masterMounts": ["Test"]}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"masterMounts": ["Test"]}]) self.assertTrue(vc.check_volumes()) self.assertTrue(vc.required_resources_dict["Volumes"] == 0) self.assertTrue(vc.required_resources_dict["VolumeGigabytes"] == 0) @@ -210,39 +212,39 @@ def test_check_volumes_match_volume(self): def test_check_network_none(self): provider = Mock() provider.get_network_by_id_or_name = MagicMock(return_value=None) - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{}]) self.assertFalse(vc.check_network()) def test_check_network_no_network(self): provider = Mock() provider.get_subnet_by_id_or_name = MagicMock(return_value="network") - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{"subnet": "subnet_name"}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"subnet": "subnet_name"}]) self.assertTrue(vc.check_network()) provider.get_subnet_by_id_or_name.assert_called_with("subnet_name") def test_check_network_no_network_mismatch_subnet(self): provider = Mock() provider.get_subnet_by_id_or_name = MagicMock(return_value=None) - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{"subnet": "subnet_name"}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"subnet": "subnet_name"}]) self.assertFalse(vc.check_network()) provider.get_subnet_by_id_or_name.assert_called_with("subnet_name") def test_check_network_no_subnet_mismatch_network(self): provider = Mock() provider.get_network_by_id_or_name = MagicMock(return_value=None) - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{"network": "network_name"}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"network": "network_name"}]) self.assertFalse(vc.check_network()) provider.get_network_by_id_or_name.assert_called_with("network_name") def test_check_network_no_subnet(self): provider = Mock() provider.get_network_by_id_or_name = MagicMock(return_value="network") - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{"network": "network_name"}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"network": "network_name"}]) self.assertTrue(vc.check_network()) provider.get_network_by_id_or_name.assert_called_with("network_name") @@ -250,31 +252,31 @@ def test_check_network_subnet_network(self): provider = Mock() provider.get_network_by_id_or_name = MagicMock(return_value="network") provider.get_subnet_by_id_or_name = MagicMock(return_value="network") - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{"network": "network_name"}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"network": "network_name"}]) self.assertTrue(vc.check_network()) provider.get_network_by_id_or_name.assert_called_with("network_name") def test_check_server_group_none(self): provider = Mock() provider.get_network_by_id_or_name = MagicMock(return_value=None) - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{}]) self.assertTrue(vc.check_server_group()) def test_check_server_group_mismatch(self): provider = Mock() provider.get_server_group_by_id_or_name = MagicMock(return_value=None) - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{"serverGroup": "GroupName"}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"serverGroup": "GroupName"}]) self.assertFalse(vc.check_server_group()) provider.get_server_group_by_id_or_name.assert_called_with("GroupName") def test_check_server_group_match(self): provider = Mock() provider.get_server_group_by_id_or_name = MagicMock(return_value="Group") - vc = validateConfiguration.ValidateConfiguration(providers=[provider], - configurations=[{"serverGroup": "GroupName"}]) + vc = validate_configuration.ValidateConfiguration(providers=[provider], + configurations=[{"serverGroup": "GroupName"}]) self.assertTrue(vc.check_server_group()) provider.get_server_group_by_id_or_name.assert_called_with("GroupName") @@ -284,14 +286,14 @@ def test_check_quotas_true(self): test_dict = {'total_cores': 42, 'floating_ips': 42, 'instances': 42, 'total_ram': 42, 'Volumes': 42, 'VolumeGigabytes': 42, 'Snapshots': 42, 'Backups': 42, 'BackupGigabytes': 42} provider.get_free_resources.return_value = test_dict - vc = validateConfiguration.ValidateConfiguration(providers=[provider], configurations=None) + vc = validate_configuration.ValidateConfiguration(providers=[provider], configurations=None) with patch.object(vc, "has_enough") as mock: mock.side_effect = [True] * len(test_dict) self.assertTrue(vc.check_quotas()) provider.get_free_resources.assert_called() for key in vc.required_resources_dict.keys(): self.assertTrue(call(test_dict[key], vc.required_resources_dict[key], - f"Project identifier", key) in mock.call_args_list) + "Project identifier", key) in mock.call_args_list) def test_check_quotas_false(self): provider = MagicMock() @@ -299,7 +301,7 @@ def test_check_quotas_false(self): 'Volumes': 42, 'VolumeGigabytes': 42, 'Snapshots': 42, 'Backups': 42, 'BackupGigabytes': 42} provider.get_free_resources.return_value = test_dict os.environ['OS_PROJECT_NAME'] = "name" - vc = validateConfiguration.ValidateConfiguration(providers=[provider], configurations=None) + vc = validate_configuration.ValidateConfiguration(providers=[provider], configurations=None) with patch.object(vc, "has_enough") as mock: mock.side_effect = [True] * (len(test_dict) - 1) + [False] self.assertFalse(vc.check_quotas()) @@ -307,13 +309,13 @@ def test_check_quotas_false(self): mock.assert_called() def test_has_enough_lower(self): - vc = validateConfiguration + vc = validate_configuration self.assertTrue(vc.has_enough(2, 1, "", "")) def test_has_enough_equal(self): - vc = validateConfiguration + vc = validate_configuration self.assertTrue(vc.has_enough(2, 2, "", "")) def test_has_enough_higher(self): - vc = validateConfiguration + vc = validate_configuration self.assertFalse(vc.has_enough(1, 2, "", "")) diff --git a/tests/test_ansibleConfigurator.py b/tests/test_ansibleConfigurator.py index 85f232725..8f524f71a 100644 --- a/tests/test_ansibleConfigurator.py +++ b/tests/test_ansibleConfigurator.py @@ -1,12 +1,13 @@ from unittest import TestCase from unittest.mock import MagicMock, Mock, patch, call, mock_open, ANY -import bibigrid2.core.utility.ansible_configurator as ansibleConfigurator -import bibigrid2.core.utility.paths.ansible_resources_path as aRP -import bibigrid2.core.utility.yaml_dumper as yamlDumper +import bibigrid.core.utility.ansible_configurator as ansibleConfigurator +import bibigrid.core.utility.paths.ansible_resources_path as aRP +import bibigrid.core.utility.yaml_dumper as yamlDumper class TestAnsibleConfigurator(TestCase): + # pylint: disable=R0904 def test_generate_site_file_yaml_empty(self): site_yaml = [{'hosts': 'master', "become": "yes", "vars_files": ansibleConfigurator.VARS_FILES, "roles": ["common", "master"]}, @@ -121,7 +122,7 @@ def test_generate_common_configuration_ide(self): self.assertEqual(common_configuration_yaml, ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)) - @patch("bibigrid2.core.utility.ansibleConfigurator.get_ansible_roles") + @patch("bibigrid.core.utility.ansibleConfigurator.get_ansible_roles") def test_generate_common_configuration_ansible_roles_mock(self, mock_ansible_roles): cidrs = 42 ansible_roles = [{elem: elem for elem in ["file", "hosts", "name", "vars", "vars_file"]}] @@ -131,7 +132,7 @@ def test_generate_common_configuration_ansible_roles_mock(self, mock_ansible_rol ansibleConfigurator.generate_common_configuration_yaml(cidrs, configuration)["ansible_roles"]) mock_ansible_roles.assert_called_with(ansible_roles) - @patch("bibigrid2.core.utility.ansibleConfigurator.get_ansible_galaxy_roles") + @patch("bibigrid.core.utility.ansibleConfigurator.get_ansible_galaxy_roles") def test_generate_common_configuration_ansible_galaxy_roles(self, mock_galaxy_roles): cidrs = 42 galaxy_roles = [{elem: elem for elem in ["hosts", "name", "galaxy", "git", "url", "vars", "vars_file"]}] @@ -142,7 +143,7 @@ def test_generate_common_configuration_ansible_galaxy_roles(self, mock_galaxy_ro "ansible_galaxy_roles"]) mock_galaxy_roles.assert_called_with(galaxy_roles) - @patch("bibigrid2.core.utility.ansibleConfigurator.to_instance_host_dict") + @patch("bibigrid.core.utility.ansibleConfigurator.to_instance_host_dict") def test_generate_ansible_hosts(self, mock_instance_host_dict): mock_instance_host_dict.side_effect = [0, 1, 2] cluster_dict = {"workers": [{"private_v4": 21}], "vpnwkrs": [{"private_v4": 32}]} @@ -268,17 +269,17 @@ def test_write_yaml_alias(self, mock_yaml): output_mock.assert_called_once_with("here", "w+") mock_yaml.assert_called_with(data={"some": "yaml"}, stream=ANY) - @patch("bibigrid2.core.utility.idGeneration.generate_munge_key") - @patch("bibigrid2.core.utility.ansibleConfigurator.generate_worker_specification_file_yaml") - @patch("bibigrid2.core.utility.ansibleConfigurator.generate_login_file_yaml") - @patch("bibigrid2.core.utility.ansibleConfigurator.generate_common_configuration_yaml") - @patch("bibigrid2.core.actions.listClusters.dict_clusters") - @patch("bibigrid2.core.utility.ansibleConfigurator.generate_instances_yaml") - @patch("bibigrid2.core.utility.ansibleConfigurator.generate_ansible_hosts_yaml") - @patch("bibigrid2.core.utility.ansibleConfigurator.get_ansible_roles") - @patch("bibigrid2.core.utility.ansibleConfigurator.generate_site_file_yaml") - @patch("bibigrid2.core.utility.ansibleConfigurator.write_yaml") - @patch("bibigrid2.core.utility.ansibleConfigurator.get_cidrs") + @patch("bibigrid.core.utility.id_generation.generate_munge_key") + @patch("bibigrid.core.utility.ansibleConfigurator.generate_worker_specification_file_yaml") + @patch("bibigrid.core.utility.ansibleConfigurator.generate_login_file_yaml") + @patch("bibigrid.core.utility.ansibleConfigurator.generate_common_configuration_yaml") + @patch("bibigrid.core.actions.list_clusters.dict_clusters") + @patch("bibigrid.core.utility.ansibleConfigurator.generate_instances_yaml") + @patch("bibigrid.core.utility.ansibleConfigurator.generate_ansible_hosts_yaml") + @patch("bibigrid.core.utility.ansibleConfigurator.get_ansible_roles") + @patch("bibigrid.core.utility.ansibleConfigurator.generate_site_file_yaml") + @patch("bibigrid.core.utility.ansibleConfigurator.write_yaml") + @patch("bibigrid.core.utility.ansibleConfigurator.get_cidrs") def test_configure_ansible_yaml(self, mock_cidrs, mock_yaml, mock_site, mock_roles, mock_hosts, mock_instances, mock_list, mock_common, mock_login, mock_worker, mock_munge): mock_munge.return_value = 420 diff --git a/tests/test_check.py b/tests/test_check.py index c77361758..b3f04f31a 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -1,8 +1,8 @@ from unittest import TestCase from unittest.mock import patch -import bibigrid2.core.actions.check as check -import bibigrid2.core.utility.validate_configuration as validateConfiguration +from bibigrid.core.actions import check +from bibigrid.core.utility import validate_configuration class TestCheck(TestCase): @@ -11,7 +11,7 @@ class TestCheck(TestCase): def test_check_true(self, mock_log): providers = [42] configurations = [32] - with patch.object(validateConfiguration.ValidateConfiguration, "validate", return_value=True) as mock_validate: + with patch.object(validate_configuration.ValidateConfiguration, "validate", return_value=True) as mock_validate: self.assertFalse(check.check(configurations, providers)) mock_validate.assert_called() mock_log.assert_called_with("Total check returned True.") @@ -20,12 +20,13 @@ def test_check_true(self, mock_log): def test_check_false(self, mock_log): providers = [42] configurations = [32] - with patch.object(validateConfiguration.ValidateConfiguration, "validate", return_value=False) as mock_validate: + with patch.object(validate_configuration.ValidateConfiguration, "validate", + return_value=False) as mock_validate: self.assertFalse(check.check(configurations, providers)) mock_validate.assert_called() mock_log.assert_called_with("Total check returned False.") - @patch("bibigrid2.core.utility.validateConfiguration.ValidateConfiguration") + @patch("bibigrid.core.utility.validate_configuration.ValidateConfiguration") def test_check_init(self, mock_validator): providers = [42] configurations = [32] diff --git a/tests/test_configurationHandler.py b/tests/test_configurationHandler.py index 8e491c004..f62161afc 100644 --- a/tests/test_configurationHandler.py +++ b/tests/test_configurationHandler.py @@ -1,13 +1,12 @@ +import os from unittest import TestCase - -import yaml - -import bibigrid2.core.utility.handler.configuration_handler as configurationHandler from unittest.mock import patch, mock_open, MagicMock -import os + +import bibigrid.core.utility.handler.configuration_handler as configurationHandler class TestConfigurationHandler(TestCase): + # pylint: disable=R0904 def test_get_list_by_name_none(self): configurations = [{}, {}] self.assertEqual([None, None], configurationHandler.get_list_by_key(configurations, "key1")) @@ -70,7 +69,7 @@ def test_find_file_in_folders_not_found_no_file(self): mock_isfile.called_with(os.path.expanduser(os.path.join("or_false_folder", "false_file"))) @patch("os.path.isfile") - @patch("bibigrid2.core.utility.handler.configurationHandler.read_configuration") + @patch("bibigrid.core.utility.handler.configurationHandler.read_configuration") def test_find_file_in_folders(self, mock_read_configuration, mock_isfile): expected_result = 42 mock_isfile.return_value(True) @@ -79,42 +78,42 @@ def test_find_file_in_folders(self, mock_read_configuration, mock_isfile): self.assertEqual(expected_result, result) mock_read_configuration.assert_called_with(os.path.expanduser(os.path.join("true_folder", "true_file"))) - @patch("bibigrid2.core.utility.handler.configurationHandler.find_file_in_folders") + @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") def test_get_cloud_files_none(self, mock_ffif): mock_ffif.return_value = None expected_result = None, None result = configurationHandler.get_clouds_files() self.assertEqual(expected_result, result) - @patch("bibigrid2.core.utility.handler.configurationHandler.find_file_in_folders") + @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") def test_get_cloud_files_no_clouds_yaml(self, mock_ffif): mock_ffif.side_effect = [None, {configurationHandler.CLOUD_PUBLIC_ROOT_KEY: 42}] expected_result = None, 42 result = configurationHandler.get_clouds_files() self.assertEqual(expected_result, result) - @patch("bibigrid2.core.utility.handler.configurationHandler.find_file_in_folders") + @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") def test_get_cloud_files_no_public_clouds_yaml(self, mock_ffif): mock_ffif.side_effect = [{configurationHandler.CLOUD_ROOT_KEY: 42}, None] expected_result = 42, None result = configurationHandler.get_clouds_files() self.assertEqual(expected_result, result) - @patch("bibigrid2.core.utility.handler.configurationHandler.find_file_in_folders") + @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") def test_get_cloud_files_no_root_key_public(self, mock_ffif): mock_ffif.side_effect = [{configurationHandler.CLOUD_ROOT_KEY: 42}, {"name": 42}] expected_result = 42, None result = configurationHandler.get_clouds_files() self.assertEqual(expected_result, result) - @patch("bibigrid2.core.utility.handler.configurationHandler.find_file_in_folders") + @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") def test_get_cloud_files_no_root_key_cloud(self, mock_ffif): mock_ffif.side_effect = [{"name": 42}, {configurationHandler.CLOUD_PUBLIC_ROOT_KEY: 42}] expected_result = None, 42 result = configurationHandler.get_clouds_files() self.assertEqual(expected_result, result) - @patch("bibigrid2.core.utility.handler.configurationHandler.find_file_in_folders") + @patch("bibigrid.core.utility.handler.configurationHandler.find_file_in_folders") def test_get_cloud_files(self, mock_ffif): mock_ffif.side_effect = [{configurationHandler.CLOUD_ROOT_KEY: 22}, {configurationHandler.CLOUD_PUBLIC_ROOT_KEY: 42}] @@ -123,8 +122,8 @@ def test_get_cloud_files(self, mock_ffif): self.assertEqual(expected_result, result) mock_ffif.assert_called_with(configurationHandler.CLOUDS_PUBLIC_YAML, configurationHandler.CLOUDS_YAML_PATHS) - @patch("bibigrid2.core.utility.handler.configurationHandler.get_cloud_specification") - @patch("bibigrid2.core.utility.handler.configurationHandler.get_clouds_files") + @patch("bibigrid.core.utility.handler.configurationHandler.get_cloud_specification") + @patch("bibigrid.core.utility.handler.configurationHandler.get_clouds_files") def test_get_cloud_specifications_none(self, mock_get_clouds_files, mock_get_clouds_specification): mock_get_clouds_files.return_value = None, None expected_result = [] @@ -133,8 +132,8 @@ def test_get_cloud_specifications_none(self, mock_get_clouds_files, mock_get_clo mock_get_clouds_specification.assert_not_called() mock_get_clouds_files.assert_called() - @patch("bibigrid2.core.utility.handler.configurationHandler.get_cloud_specification") - @patch("bibigrid2.core.utility.handler.configurationHandler.get_clouds_files") + @patch("bibigrid.core.utility.handler.configurationHandler.get_cloud_specification") + @patch("bibigrid.core.utility.handler.configurationHandler.get_clouds_files") def test_get_cloud_specifications_no_cloud_configuration_key(self, mock_get_clouds_files, mock_get_clouds_specification): mock_get_clouds_files.return_value = {"Some"}, {"Some"} @@ -144,8 +143,8 @@ def test_get_cloud_specifications_no_cloud_configuration_key(self, mock_get_clou mock_get_clouds_specification.assert_not_called() mock_get_clouds_files.assert_called() - @patch("bibigrid2.core.utility.handler.configurationHandler.get_cloud_specification") - @patch("bibigrid2.core.utility.handler.configurationHandler.get_clouds_files") + @patch("bibigrid.core.utility.handler.configurationHandler.get_cloud_specification") + @patch("bibigrid.core.utility.handler.configurationHandler.get_clouds_files") def test_get_cloud_specifications_cloud(self, mock_get_clouds_files, mock_get_clouds_specification): mock_get_clouds_files.return_value = {"1"}, {"2"} mock_get_clouds_specification.return_value = 21 @@ -155,8 +154,8 @@ def test_get_cloud_specifications_cloud(self, mock_get_clouds_files, mock_get_cl mock_get_clouds_specification.assert_called_with(42, {"1"}, {"2"}) mock_get_clouds_files.assert_called() - @patch("bibigrid2.core.utility.handler.configurationHandler.get_cloud_specification") - @patch("bibigrid2.core.utility.handler.configurationHandler.get_clouds_files") + @patch("bibigrid.core.utility.handler.configurationHandler.get_cloud_specification") + @patch("bibigrid.core.utility.handler.configurationHandler.get_clouds_files") def test_get_cloud_specifications_no_config(self, mock_get_clouds_files, mock_get_clouds_specification): mock_get_clouds_files.return_value = {"1"}, {"2"} mock_get_clouds_specification.return_value = 21 diff --git a/tests/test_create.py b/tests/test_create.py index 3cc45257e..e3cd9068e 100644 --- a/tests/test_create.py +++ b/tests/test_create.py @@ -2,12 +2,13 @@ from unittest import TestCase from unittest.mock import patch, Mock, MagicMock, mock_open -import bibigrid2.core.actions.create as create +from bibigrid.core.actions import create class TestCreate(TestCase): - @patch("bibigrid2.core.utility.handler.sshHandler.get_add_ssh_public_key_commands") - @patch("bibigrid2.core.utility.idGeneration.generate_safe_cluster_id") + # pylint: disable=R0904 + @patch("bibigrid.core.utility.handler.sshHandler.get_add_ssh_public_key_commands") + @patch("bibigrid.core.utility.id_generation.generate_safe_cluster_id") def test_init(self, mock_id, mock_ssh): unique_id = 21 provider = MagicMock() @@ -23,8 +24,8 @@ def test_init(self, mock_id, mock_ssh): self.assertEqual(c.key_name, key_name) mock_id.assert_called_with([provider]) - @patch("bibigrid2.core.utility.handler.sshHandler.get_add_ssh_public_key_commands") - @patch("bibigrid2.core.utility.idGeneration.generate_safe_cluster_id") + @patch("bibigrid.core.utility.handler.sshHandler.get_add_ssh_public_key_commands") + @patch("bibigrid.core.utility.id_generation.generate_safe_cluster_id") def test_init_username(self, mock_id, mock_ssh): unique_id = 21 mock_id.return_value = str(unique_id) @@ -71,14 +72,15 @@ def test_start_instance_worker(self): network = 21 c.start_instance(provider, create.WORKER_IDENTIFIER, server_type, network, worker=True, volumes=None, external_network=None) - provider.create_server.assert_called_with(name=create.WORKER_IDENTIFIER.format(0) + create.SEPARATOR + c.cluster_id, - flavor=server_type["type"], - key_name=c.key_name, - image=server_type["image"], - network=network, volumes=None) + provider.create_server.assert_called_with( + name=create.WORKER_IDENTIFIER.format(0) + create.SEPARATOR + c.cluster_id, + flavor=server_type["type"], + key_name=c.key_name, + image=server_type["image"], + network=network, volumes=None) provider.create_floating_ip.assert_not_called() - @patch("bibigrid2.models.returnThreading.ReturnThread") + @patch("bibigrid.models.returnThreading.ReturnThread") def test_start_instances(self, return_mock): provider = MagicMock() provider.list_servers.return_value = [] @@ -96,7 +98,7 @@ def test_start_instances(self, return_mock): args=[provider, 0, 1, configuration["network"], False, 2, 32]) @patch("threading.Thread") - @patch("bibigrid2.models.returnThreading.ReturnThread") + @patch("bibigrid.models.returnThreading.ReturnThread") def test_start_instances_workers(self, return_mock, thread_mock): provider = MagicMock() provider.list_servers.return_value = [] @@ -153,7 +155,7 @@ def test_prepare_args_keyerror(self): c.prepare_vpn_or_master_args(configuration, provider)) prepare_mock.assert_not_called() - @patch("bibigrid2.core.utility.handler.sshHandler.ansible_preparation") + @patch("bibigrid.core.utility.handler.sshHandler.ansible_preparation") def test_setup_reachable_servers_master(self, mock_ansible): provider = MagicMock() provider.list_servers.return_value = [] @@ -237,8 +239,8 @@ def test_prepare_configurations_none(self): with self.assertRaises(KeyError): c.prepare_configurations() - @patch("bibigrid2.core.utility.ansibleConfigurator.configure_ansible_yaml") - @patch("bibigrid2.core.utility.handler.sshHandler.execute_ssh") + @patch("bibigrid.core.utility.ansibleConfigurator.configure_ansible_yaml") + @patch("bibigrid.core.utility.handler.sshHandler.execute_ssh") def test_upload_playbooks(self, mock_ssh, mock_configure_ansible): provider = MagicMock() provider.list_servers.return_value = [] @@ -250,8 +252,9 @@ def test_upload_playbooks(self, mock_ssh, mock_configure_ansible): configurations=c.configurations, cluster_id=c.cluster_id) mock_ssh.assert_called_with(floating_ip=c.master_ip, private_key=create.KEY_FOLDER + c.key_name, - username=c.ssh_user, filepaths=[(os.path.expanduser("/Documents/Repos/bibigrid2/" - "resources/playbook/"), "playbook")], + username=c.ssh_user, filepaths=[(os.path.expanduser("/Documents/Repos/bibigrid/" + "resources/playbook/"), + "playbook")], commands=['echo ansible_start']) @patch("threading.Thread") @@ -272,7 +275,7 @@ def test_start_start_instances_thread(self, mock_thread): @patch.object(create.Create, "start_start_instances_threads") @patch.object(create.Create, "upload_data") @patch.object(create.Create, "print_cluster_start_info") - @patch("bibigrid2.core.actions.terminateCluster.terminate_cluster") + @patch("bibigrid.core.actions.terminateCluster.terminate_cluster") def test_create_non_debug(self, mock_terminate, mock_info, mock_up, mock_start, mock_conf, mock_key): provider = MagicMock() provider.list_servers.return_value = [] @@ -288,7 +291,7 @@ def test_create_non_debug(self, mock_terminate, mock_info, mock_up, mock_start, @patch.object(create.Create, "start_start_instances_threads") @patch.object(create.Create, "upload_data") @patch.object(create.Create, "print_cluster_start_info") - @patch("bibigrid2.core.actions.terminateCluster.terminate_cluster") + @patch("bibigrid.core.actions.terminateCluster.terminate_cluster") def test_create_non_debug_upload_raise(self, mock_terminate, mock_info, mock_up, mock_start, mock_conf, mock_key): provider = MagicMock() provider.list_servers.return_value = [] @@ -307,7 +310,7 @@ def test_create_non_debug_upload_raise(self, mock_terminate, mock_info, mock_up, @patch.object(create.Create, "start_start_instances_threads") @patch.object(create.Create, "upload_data") @patch.object(create.Create, "print_cluster_start_info") - @patch("bibigrid2.core.actions.terminateCluster.terminate_cluster") + @patch("bibigrid.core.actions.terminateCluster.terminate_cluster") def test_create_debug(self, mock_terminate, mock_info, mock_up, mock_start, mock_conf, mock_key): provider = MagicMock() provider.list_servers.return_value = [] diff --git a/tests/test_idGeneration.py b/tests/test_idGeneration.py index f139772e0..6e6945308 100644 --- a/tests/test_idGeneration.py +++ b/tests/test_idGeneration.py @@ -1,28 +1,28 @@ from unittest import TestCase from unittest.mock import Mock, MagicMock, patch -import bibigrid2.core.actions.create as create -import bibigrid2.core.utility.id_generation as idGeneration +from bibigrid.core.actions import create +from bibigrid.core.utility import id_generation class Test(TestCase): def test_generate_cluster_id(self): """ - This test is not ideal, but prevents worst changes within a reasonable runtime + This test is not ideal, but prevents worst within a reasonable runtime :return: """ test_list = [] - for x in range(10000): - test_list.append(idGeneration.generate_cluster_id()) + for _ in range(10000): + test_list.append(id_generation.generate_cluster_id()) self.assertTrue(len(set(test_list)) == len(test_list)) - @patch("bibigrid2.core.utility.idGeneration.generate_cluster_id") + @patch("bibigrid.core.utility.id_generation.generate_cluster_id") def test_generate_safe_cluster_id(self, mock_generate_cluster_id): mock_generate_cluster_id.return_value = 21 - with patch("bibigrid2.core.utility.idGeneration.is_unique_cluster_id") as mock_is_unique: + with patch("bibigrid.core.utility.id_generation.is_unique_cluster_id") as mock_is_unique: mock_is_unique.side_effect = [True] - self.assertTrue(idGeneration.generate_safe_cluster_id([42])) + self.assertTrue(id_generation.generate_safe_cluster_id([42])) mock_is_unique.assert_called_with(21, [42]) def test_is_unique_cluster_id_duplicate(self): @@ -30,7 +30,7 @@ def test_is_unique_cluster_id_duplicate(self): provider = Mock() provider.list_servers = MagicMock( return_value=[{"name": create.MASTER_IDENTIFIER + create.SEPARATOR + str(cluster_id)}]) - self.assertFalse(idGeneration.is_unique_cluster_id(str(cluster_id), [provider])) + self.assertFalse(id_generation.is_unique_cluster_id(str(cluster_id), [provider])) provider.list_servers.assert_called() def test_is_unique_cluster_id_unique(self): @@ -38,5 +38,5 @@ def test_is_unique_cluster_id_unique(self): provider = Mock() provider.list_servers = MagicMock( return_value=[{"name": create.MASTER_IDENTIFIER + create.SEPARATOR + str(cluster_id + 1)}]) - self.assertTrue(idGeneration.is_unique_cluster_id(str(cluster_id), [provider])) + self.assertTrue(id_generation.is_unique_cluster_id(str(cluster_id), [provider])) provider.list_servers.assert_called() diff --git a/tests/test_listClusters.py b/tests/test_listClusters.py index 4c1fdfd3a..7a272c1fe 100644 --- a/tests/test_listClusters.py +++ b/tests/test_listClusters.py @@ -1,8 +1,8 @@ from unittest import TestCase from unittest.mock import Mock -import bibigrid2.core.actions.create as create -import bibigrid2.core.actions.list_clusters as listClusters +from bibigrid.core.actions import create +from bibigrid.core.actions import list_clusters class TestDictClusters(TestCase): @@ -14,8 +14,8 @@ def test_setup(self): cluster_dict = {} server = {"name": identifier + create.SEPARATOR + str(cluster_id)} self.assertEqual(str(cluster_id), - listClusters.setup(server, - identifier, cluster_dict, test_provider)) + list_clusters.setup(server, + identifier, cluster_dict, test_provider)) self.assertEqual({str(cluster_id): {'worker': [], 'vpnwkr': []}}, cluster_dict) self.assertEqual(test_provider, server["provider"]) @@ -27,8 +27,8 @@ def test_setup_already(self): cluster_dict = {str(cluster_id): {'worker': ["some"], 'vpnwkr': ["some"]}} server = {"name": identifier + create.SEPARATOR + str(cluster_id)} self.assertEqual(str(cluster_id), - listClusters.setup(server, - identifier, cluster_dict, test_provider)) + list_clusters.setup(server, + identifier, cluster_dict, test_provider)) self.assertEqual({str(cluster_id): {'worker': ["some"], 'vpnwkr': ["some"]}}, cluster_dict) self.assertEqual(test_provider, server["provider"]) @@ -43,4 +43,4 @@ def test_dict_clusters(self): in [create.WORKER_IDENTIFIER, create.VPN_WORKER_IDENTIFIER, create.MASTER_IDENTIFIER]] - self.assertEqual(expected, listClusters.dict_clusters([provider])) + self.assertEqual(expected, list_clusters.dict_clusters([provider])) diff --git a/tests/test_providerHandler.py b/tests/test_providerHandler.py index e6eeab17f..6735e77f4 100644 --- a/tests/test_providerHandler.py +++ b/tests/test_providerHandler.py @@ -1,18 +1,18 @@ from unittest import TestCase from unittest.mock import MagicMock, patch -import bibigrid2.core.utility.handler.provider_handler as providerHandler +import bibigrid.core.utility.handler.provider_handler as providerHandler class TestProviderHandler(TestCase): - @patch("bibigrid2.core.utility.handler.configurationHandler.get_cloud_specifications") - @patch("bibigrid2.core.utility.handler.providerHandler.get_provider_list_by_name_list") + @patch("bibigrid.core.utility.handler.configurationHandler.get_cloud_specifications") + @patch("bibigrid.core.utility.handler.providerHandler.get_provider_list_by_name_list") def test_get_providers(self, mock_provider_list, mock_get_cloud_specifications): - mock_get_cloud_specifications.return_value = True # for if not false + mock_get_cloud_specifications.return_value = True # for if not false configurations = [{"infrastructure": "some"}] mock_provider_list.return_value = 42 - with patch("bibigrid2.core.utility.handler.configurationHandler.get_list_by_key") as mock_by_name: + with patch("bibigrid.core.utility.handler.configurationHandler.get_list_by_key") as mock_by_name: self.assertEqual(42, providerHandler.get_providers(configurations)) mock_by_name.assert_called_with(configurations, "infrastructure") mock_get_cloud_specifications.assert_called_with(configurations) @@ -20,7 +20,7 @@ def test_get_providers(self, mock_provider_list, mock_get_cloud_specifications): def test_get_provider_list_by_name_list(self): keys = providerHandler.PROVIDER_NAME_DICT.keys() values = [42] - with patch("bibigrid2.core.utility.handler.providerHandler.get_provider_by_name") as mock_by_name: + with patch("bibigrid.core.utility.handler.providerHandler.get_provider_by_name") as mock_by_name: mock_by_name.return_value = MagicMock(return_value=42) self.assertEqual(providerHandler.get_provider_list_by_name_list(keys, "nonempty_specification"), values) mock_by_name.assert_called_with(list(keys)[0]) diff --git a/tests/test_returnThreading.py b/tests/test_returnThreading.py index 9256280c4..be9eeecdc 100644 --- a/tests/test_returnThreading.py +++ b/tests/test_returnThreading.py @@ -1,6 +1,6 @@ from unittest import TestCase -import bibigrid2.models.return_threading as returnThreading +import bibigrid.models.return_threading as returnThreading def test_method(x): diff --git a/tests/test_sshHandler.py b/tests/test_sshHandler.py index 54c7dda6d..7a9f05df9 100644 --- a/tests/test_sshHandler.py +++ b/tests/test_sshHandler.py @@ -4,7 +4,7 @@ from paramiko.ssh_exception import NoValidConnectionsError -import bibigrid2.core.utility.handler.ssh_handler as sshHandler +import bibigrid.core.utility.handler.ssh_handler as sshHandler class TestSshHandler(TestCase): @@ -62,7 +62,7 @@ def test_is_active_exception(self, mock_log): client.connect.assert_called_with(hostname=42, username=22, pkey=32) mock_log.assert_called() - @patch("bibigrid2.core.utility.handler.sshHandler.execute_ssh_cml_commands") + @patch("bibigrid.core.utility.handler.sshHandler.execute_ssh_cml_commands") @patch("paramiko.ECDSAKey.from_private_key_file") @patch("paramiko.SSHClient") def test_execute_ssh(self, mock_client, mock_paramiko_key, mock_exec): @@ -72,19 +72,19 @@ def test_execute_ssh(self, mock_client, mock_paramiko_key, mock_exec): mock_client.return_value = mock mock.__enter__ = client mock.__exit__ = Mock(return_value=None) - with patch("bibigrid2.core.utility.handler.sshHandler.is_active") as mock_active: + with patch("bibigrid.core.utility.handler.sshHandler.is_active") as mock_active: sshHandler.execute_ssh(42, 32, 22, [12], None) mock_client.assert_called_with() mock_active.assert_called_with(client=client(), floating_ip_address=42, username=22, private_key=2) mock_exec.assert_called_with(client(), [12]) mock_paramiko_key.assert_called_with(32) - @patch("bibigrid2.core.utility.handler.sshHandler.execute_ssh") + @patch("bibigrid.core.utility.handler.sshHandler.execute_ssh") def test_ansible_preparation(self, mock_execute): sshHandler.ansible_preparation(1, 2, 3, [], []) mock_execute.assert_called_with(1, 2, 3, [] + sshHandler.ANSIBLE_SETUP, [(2, sshHandler.PRIVATE_KEY_FILE)]) - @patch("bibigrid2.core.utility.handler.sshHandler.execute_ssh") + @patch("bibigrid.core.utility.handler.sshHandler.execute_ssh") def test_ansible_preparation_elem(self, mock_execute): sshHandler.ansible_preparation(1, 2, 3, [42], [42]) mock_execute.assert_called_with(1, 2, 3, sshHandler.ANSIBLE_SETUP + [42], diff --git a/tests/test_startup.py b/tests/test_startup.py index 7138c808b..3baee61a9 100644 --- a/tests/test_startup.py +++ b/tests/test_startup.py @@ -1,11 +1,11 @@ from unittest import TestCase from unittest.mock import Mock, patch, MagicMock -import bibigrid2.core.startup as startup +from bibigrid.core import startup class TestStartup(TestCase): - @patch('bibigrid2.core.utility.handler.providerHandler.get_providers') + @patch('bibigrid.core.utility.handler.providerHandler.get_providers') def test_provider(self, mock_get_providers): args = Mock() args.list_clusters = True @@ -15,13 +15,13 @@ def test_provider(self, mock_get_providers): provider.close = MagicMock() configurations = {} mock_get_providers.return_value = [provider] - with patch("bibigrid2.core.actions.listClusters.print_list_clusters") as mock_lc: + with patch("bibigrid.core.actions.list_clusters.print_list_clusters") as mock_lc: mock_lc.return_value = 42 self.assertTrue(startup.run_action(args, configurations, "") == 42) mock_get_providers.assert_called_with(configurations) provider.close.assert_called() - @patch('bibigrid2.core.utility.handler.providerHandler.get_providers') + @patch('bibigrid.core.utility.handler.providerHandler.get_providers') def test_list_clusters(self, get_providers): provider_mock = Mock() provider_mock.close = Mock() @@ -31,12 +31,12 @@ def test_list_clusters(self, get_providers): args.version = False args.cluster_id = 12 configurations = {} - with patch("bibigrid2.core.actions.listClusters.print_list_clusters") as mock_lc: + with patch("bibigrid.core.actions.list_clusters.print_list_clusters") as mock_lc: mock_lc.return_value = 42 self.assertTrue(startup.run_action(args, configurations, "") == 42) mock_lc.assert_called_with(12, [provider_mock]) - @patch('bibigrid2.core.utility.handler.providerHandler.get_providers') + @patch('bibigrid.core.utility.handler.providerHandler.get_providers') def test_check(self, get_providers): provider_mock = Mock() provider_mock.close = Mock() @@ -47,13 +47,13 @@ def test_check(self, get_providers): args.check = True args.cluster_id = 12 configurations = {} - with patch("bibigrid2.core.actions.check.check") as mock_lc: + with patch("bibigrid.core.actions.check.check") as mock_lc: mock_lc.return_value = 42 self.assertTrue(startup.run_action(args, configurations, "") == 42) mock_lc.assert_called_with(configurations, [provider_mock]) - @patch('bibigrid2.core.utility.handler.providerHandler.get_providers') - @patch('bibigrid2.core.actions.create.Create') + @patch('bibigrid.core.utility.handler.providerHandler.get_providers') + @patch('bibigrid.core.actions.create.Create') def test_create(self, mock_create, get_providers): provider_mock = Mock() provider_mock.close = Mock() @@ -70,10 +70,11 @@ def test_create(self, mock_create, get_providers): creator.create = MagicMock(return_value=42) mock_create.return_value = creator self.assertTrue(startup.run_action(args, configurations, "") == 42) - mock_create.assert_called_with(providers=[provider_mock], configurations=configurations, debug=True, config_path="") + mock_create.assert_called_with(providers=[provider_mock], configurations=configurations, debug=True, + config_path="") creator.create.assert_called() - @patch('bibigrid2.core.utility.handler.providerHandler.get_providers') + @patch('bibigrid.core.utility.handler.providerHandler.get_providers') def test_terminate(self, get_providers): provider_mock = Mock() provider_mock.close = Mock() @@ -87,13 +88,13 @@ def test_terminate(self, get_providers): args.cluster_id = 12 args.debug = True configurations = {} - with patch("bibigrid2.core.actions.terminateCluster.terminate_cluster") as mock_tc: + with patch("bibigrid.core.actions.terminateCluster.terminate_cluster") as mock_tc: mock_tc.return_value = 42 self.assertTrue(startup.run_action(args, configurations, "") == 42) mock_tc.assert_called_with(12, [provider_mock], True) - @patch('bibigrid2.core.utility.handler.providerHandler.get_providers') - @patch("bibigrid2.core.actions.ide.ide") + @patch('bibigrid.core.utility.handler.providerHandler.get_providers') + @patch("bibigrid.core.actions.ide.ide") def test_ide(self, mock_ide, get_providers): provider_mock = Mock() provider_mock.close = Mock() diff --git a/tests/test_terminateCluster.py b/tests/test_terminateCluster.py index 68f0f5474..4d9348a69 100644 --- a/tests/test_terminateCluster.py +++ b/tests/test_terminateCluster.py @@ -1,16 +1,14 @@ -import os from unittest import TestCase -from unittest.mock import MagicMock, Mock, patch, call +from unittest.mock import MagicMock, patch -import bibigrid2.core.actions.create as create -import bibigrid2.core.actions.terminate_cluster as terminateCluster +from bibigrid.core.actions import create +from bibigrid.core.actions import terminate_cluster class TestTerminate(TestCase): - @patch("bibigrid2.core.actions.terminateCluster.terminate_output") - @patch("logging.info") - def test_terminate_cluster(self, mock_log, mock_output): + @patch("bibigrid.core.actions.terminate_cluster.terminate_output") + def test_terminate_cluster(self, _, mock_output): provider = MagicMock() provider.cloud_specification["auth"]["project_name"] = 32 cluster_id = 42 @@ -18,21 +16,22 @@ def test_terminate_cluster(self, mock_log, mock_output): {"name": create.MASTER_IDENTIFIER + create.SEPARATOR + str(cluster_id), "id": 21}] provider.delete_server.return_value = True provider.delete_keypair.return_value = True - terminateCluster.terminate_cluster(str(cluster_id), [provider], False) + terminate_cluster.terminate_cluster(str(cluster_id), [provider], False) provider.delete_server.assert_called_with(21) provider.delete_keypair.assert_called_with( create.KEY_PREFIX + provider.cloud_specification["auth"]["project_name"] + create.SEPARATOR + str(cluster_id)) mock_output.assert_called_with([provider.delete_server.return_value], [provider.delete_keypair.return_value], str(cluster_id)) + @patch("logging.info") - def test_terminate_cluster_none(self, mock_log): + def test_terminate_cluster_none(self, _): provider = MagicMock() provider[0].specification["auth"]["project_name"] = "test_project_name" cluster_id = 42 provider.list_servers.return_value = [ {"name": create.MASTER_IDENTIFIER + create.SEPARATOR + str(cluster_id + 1), "id": 21}] provider.delete_keypair.return_value = False - terminateCluster.terminate_cluster(str(cluster_id), [provider], False) + terminate_cluster.terminate_cluster(str(cluster_id), [provider], False) provider.delete_server.assert_not_called() - provider.delete_keypair.assert_called_with('bibigrid42') # since keypair is not called + provider.delete_keypair.assert_called_with('bibigrid42') # since keypair is not called