diff --git a/.github/workflows/blacked.yml b/.github/workflows/blacked.yml index af48ff08..c6c1686d 100644 --- a/.github/workflows/blacked.yml +++ b/.github/workflows/blacked.yml @@ -4,9 +4,19 @@ jobs: format: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3.5.0 - - name: Checking out Branch - run: git checkout "${GITHUB_REF:11}" + - uses: actions/checkout@v3 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Extract branch name + shell: bash + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + id: extract_branch + - name: Set tag + run: sed 's/\//-/g' <<< "::set-output name=TAG::${{ steps.extract_branch.outputs.branch }}" + id: tag + - name: Get tag + run: echo "The selected tag is ${{ steps.tag.outputs.TAG }}" + - name: black uses: lgeiger/black-action@master with: @@ -17,7 +27,7 @@ jobs: - name: Create Pull Request if: steps.git-check.outputs.modified == 'true' id: cpr - uses: peter-evans/create-pull-request@v4.2.4 + uses: peter-evans/create-pull-request@v5.0.0 with: token: ${{ secrets.GITHUB_TOKEN }} commit-message: fix(Linting):blacked code @@ -30,7 +40,7 @@ jobs: [1]: https://github.com/peter-evans/create-pull-request labels: automated,linting reviewers: ${{github.actor}} - branch: fix(linting)_${{github.ref}} + branch: fix(linting)_${{ steps.tag.outputs.TAG }} - name: Check outputs run: | echo "Pull Request Number - ${{ env.PULL_REQUEST_NUMBER }}" diff --git a/.github/workflows/build_image.yml b/.github/workflows/build_image.yml index 6536eca7..84b41fe8 100644 --- a/.github/workflows/build_image.yml +++ b/.github/workflows/build_image.yml @@ -8,5 +8,5 @@ jobs: uses: rokroskar/workflow-run-cleanup-action@v0.3.3 env: GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}" - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v3.5.2 - run: docker build --no-cache . diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 2d3c4def..776774d9 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -35,7 +35,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v3.5.0 + uses: actions/checkout@v3.5.2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/master-protection.yml b/.github/workflows/master-protection.yml index 9829d96e..57bf3072 100644 --- a/.github/workflows/master-protection.yml +++ b/.github/workflows/master-protection.yml @@ -14,7 +14,7 @@ jobs: uses: rokroskar/workflow-run-cleanup-action@v0.3.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v3.5.2 - name: Check Tag run: | diff --git a/.github/workflows/publish_docker.yml b/.github/workflows/publish_docker.yml index 4296d56a..e3aa2297 100644 --- a/.github/workflows/publish_docker.yml +++ b/.github/workflows/publish_docker.yml @@ -15,7 +15,7 @@ jobs: uses: rokroskar/workflow-run-cleanup-action@v0.3.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v3.5.2 - name: Extract branch name shell: bash run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" diff --git a/.github/workflows/release_image.yml b/.github/workflows/release_image.yml index 7f32142a..413b277e 100644 --- a/.github/workflows/release_image.yml +++ b/.github/workflows/release_image.yml @@ -11,7 +11,7 @@ jobs: - name: Get the version id: get_version run: echo ::set-output name=VERSION::${GITHUB_REF/refs\/tags\//} - - uses: actions/checkout@v3.5.0 + - uses: actions/checkout@v3.5.2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 - name: Login to Quay.io diff --git a/Dockerfile b/Dockerfile index c6d96053..40912b2c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.11.2-buster +FROM python:3.11.3-buster RUN apt-get update -y RUN apt-get install -y build-essential WORKDIR /code diff --git a/VirtualMachineService/VirtualMachineHandler.py b/VirtualMachineService/VirtualMachineHandler.py index 116bfa09..0b678a20 100644 --- a/VirtualMachineService/VirtualMachineHandler.py +++ b/VirtualMachineService/VirtualMachineHandler.py @@ -280,7 +280,10 @@ def __init__(self, config): try: self.RE_BACKEND_URL = cfg["forc"]["forc_url"] backend_url_host = self.RE_BACKEND_URL.split(":") - self.FORC_URL = cfg["forc"].get("openresty_url", None) or f"https:{backend_url_host[1]}/" + self.FORC_URL = ( + cfg["forc"].get("openresty_url", None) + or f"https:{backend_url_host[1]}/" + ) self.FORC_API_KEY = os.environ.get("FORC_API_KEY", None) self.FORC_ALLOWED = {} @@ -288,9 +291,9 @@ def __init__(self, config): self.FORC_REMOTE_ID = cfg["forc"]["forc_remote_id"] self.GITHUB_PLAYBOOKS_REPO = cfg["forc"]["github_playbooks_repo"] if ( - not self.RE_BACKEND_URL - or not self.FORC_API_KEY - or not self.GITHUB_PLAYBOOKS_REPO + not self.RE_BACKEND_URL + or not self.FORC_API_KEY + or not self.GITHUB_PLAYBOOKS_REPO ): raise ValueError LOG.info(msg=f"Forc-Backend url loaded: {self.RE_BACKEND_URL}") @@ -435,10 +438,10 @@ def get_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): properties = img.get("properties") if not properties: @@ -448,7 +451,7 @@ def get_Images(self): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") continue @@ -489,7 +492,7 @@ def prepare_image(self, img): tags = img.get("tags", []) LOG.info(set(self.ALL_TEMPLATES).intersection(tags)) if len( - set(self.ALL_TEMPLATES).intersection(tags) + set(self.ALL_TEMPLATES).intersection(tags) ) > 0 and not self.cross_check_forc_image(tags): LOG.info(f"Resenv check: Skipping {img['name']}.") return None @@ -527,11 +530,11 @@ def get_public_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "public", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "public", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -553,11 +556,11 @@ def get_private_Images(self): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active" - and x["visibility"] == "private", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active" + and x["visibility"] == "private", + self.conn.list_images(), ): image = self.prepare_image(img) if image is None: @@ -615,10 +618,10 @@ def get_Images_by_filter(self, filter_list): images = list() try: for img in filter( - lambda x: "tags" in x - and len(x["tags"]) > 0 - and x["status"] == "active", - self.conn.list_images(), + lambda x: "tags" in x + and len(x["tags"]) > 0 + and x["status"] == "active", + self.conn.list_images(), ): tags = img.get("tags", []) if "resenv" in filter_list: @@ -804,7 +807,7 @@ def create_add_keys_script(self, keys): return key_script def create_mount_init_script( - self, volume_ids_path_new=None, volume_ids_path_attach=None + self, volume_ids_path_new=None, volume_ids_path_attach=None ): LOG.info(f"create init script for volume ids:{volume_ids_path_new}") if not volume_ids_path_new and not volume_ids_path_attach: @@ -949,17 +952,17 @@ def create_volume(self, volume_name, volume_storage, metadata): raise ressourceException(Reason=str(e)) def volume_ids( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new, - volume_ids_path_attach, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new, + volume_ids_path_attach, ): image = self.get_image(image=image) flavor = self.get_flavor(flavor=flavor) @@ -1005,9 +1008,7 @@ def volume_ids( return {} def _validate_forc_security_group(self): - LOG.info( - f"Validate Forc Security Group..." - ) + LOG.info(f"Validate Forc Security Group...") sec = self.conn.get_security_group(name_or_id=self.FORC_REMOTE_ID) if not sec: LOG.exception(f"Forc Security Group - [{self.FORC_REMOTE_ID}] not found!") @@ -1041,7 +1042,7 @@ def get_or_create_project_security_group(self, project_name, project_id): return new_security_group["id"] def get_research_environment_security_groups( - self, research_environment_names: list[str] + self, research_environment_names: list[str] ): custom_security_groups = [] @@ -1062,18 +1063,18 @@ def get_research_environment_security_groups( return custom_security_groups def start_server_without_playbook( - self, - flavor, - image, - public_key, - servername, - metadata, - https, - http, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, - additional_keys=None, + self, + flavor, + image, + public_key, + servername, + metadata, + https, + http, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, + additional_keys=None, ): """ Start a new Server. @@ -1134,17 +1135,17 @@ def start_server_without_playbook( if init_script: add_key_script = self.create_add_keys_script(keys=additional_keys) init_script = ( - add_key_script - + encodeutils.safe_encode("\n".encode("utf-8")) - + unlock_ubuntu_user_script - + init_script + add_key_script + + encodeutils.safe_encode("\n".encode("utf-8")) + + unlock_ubuntu_user_script + + init_script ) else: init_script = ( - self.create_add_keys_script(keys=additional_keys) - + encodeutils.safe_encode("\n".encode("utf-8")) - + unlock_ubuntu_user_script + self.create_add_keys_script(keys=additional_keys) + + encodeutils.safe_encode("\n".encode("utf-8")) + + unlock_ubuntu_user_script ) server = self.conn.create_server( @@ -1170,17 +1171,17 @@ def start_server_without_playbook( return {} def start_server( - self, - flavor, - image, - public_key, - servername, - metadata, - diskspace, - volumename, - https, - http, - resenv, + self, + flavor, + image, + public_key, + servername, + metadata, + diskspace, + volumename, + https, + http, + resenv, ): """ Start a new Server. @@ -1240,7 +1241,7 @@ def start_server( return {} def create_resenv_security_group_and_attach_to_server( - self, server_id: str, resenv_template: str + self, server_id: str, resenv_template: str ): LOG.info(f"Create {resenv_template} Security Group for Instance: {server_id}") @@ -1274,16 +1275,16 @@ def create_resenv_security_group_and_attach_to_server( ) def start_server_with_custom_key( - self, - flavor, - image, - servername, - metadata, - http, - https, - resenv, - volume_ids_path_new=None, - volume_ids_path_attach=None, + self, + flavor, + image, + servername, + metadata, + http, + https, + resenv, + volume_ids_path_new=None, + volume_ids_path_attach=None, ): """ Start a new Server. @@ -1364,7 +1365,7 @@ def start_server_with_custom_key( return {} def create_and_deploy_playbook( - self, public_key, playbooks_information, openstack_id + self, public_key, playbooks_information, openstack_id ): global active_playbooks LOG.info( @@ -1418,8 +1419,8 @@ def cross_check_forc_image(self, tags): cross_tags = list(set(self.ALL_TEMPLATES).intersection(tags)) for template_dict in templates: if ( - template_dict["name"] in self.FORC_ALLOWED - and template_dict["name"] in cross_tags + template_dict["name"] in self.FORC_ALLOWED + and template_dict["name"] in cross_tags ): if template_dict["version"] in self.FORC_ALLOWED[template_dict["name"]]: return True @@ -2186,19 +2187,19 @@ def create_deactivate_update_script(self): return deactivate_update_script def add_cluster_machine( - self, - cluster_id, - cluster_user, - cluster_group_id, - image, - flavor, - name, - key_name, - batch_idx, - worker_idx, - pub_key, - project_name, - project_id, + self, + cluster_id, + cluster_user, + cluster_group_id, + image, + flavor, + name, + key_name, + batch_idx, + worker_idx, + pub_key, + project_name, + project_id, ): LOG.info( f"Add machine to [{name}] {cluster_id} - [Image: {image}] - {key_name}" @@ -2536,9 +2537,9 @@ def delete_server(self, openstack_id): return False task_state = self.check_server_task_state(openstack_id) if ( - task_state == "image_snapshot" - or task_state == "image_pending_upload" - or task_state == "image_uploading" + task_state == "image_snapshot" + or task_state == "image_pending_upload" + or task_state == "image_uploading" ): raise ConflictException("task_state in image creating") security_groups = self.conn.list_server_security_groups(server=server) @@ -2549,11 +2550,11 @@ def delete_server(self, openstack_id): ) if ( - sg["name"] != self.DEFAULT_SECURITY_GROUP_NAME - and "bibigrid" not in sg["name"] - and not self.is_security_group_in_use( - security_group_id=sg["id"] - ) + sg["name"] != self.DEFAULT_SECURITY_GROUP_NAME + and "bibigrid" not in sg["name"] + and not self.is_security_group_in_use( + security_group_id=sg["id"] + ) ): LOG.info(f"Delete security group {sg['name']}") @@ -2724,15 +2725,15 @@ def create_or_get_default_ssh_security_group(self): ) def create_security_group( - self, - name, - udp_port=None, - ssh=True, - http=False, - https=False, - udp=False, - description=None, - resenv=[], + self, + name, + udp_port=None, + ssh=True, + http=False, + https=False, + udp=False, + description=None, + resenv=[], ): LOG.info(f"Create new security group {name}") sec = self.conn.get_security_group(name_or_id=name) @@ -2917,7 +2918,7 @@ def update_playbooks(self): name for name in os.listdir(PLAYBOOKS_DIR) if name not in ["optional", "packer", ".github", "cluster"] - and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) + and os.path.isdir(os.path.join(PLAYBOOKS_DIR, name)) ] LOG.info(self.ALL_TEMPLATES) @@ -2969,7 +2970,7 @@ def load_resenv_metadata(self): for template in self.ALL_TEMPLATES: try: with open( - f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" + f"{PLAYBOOKS_DIR}{template}/{template}_metadata.yml" ) as template_metadata: try: loaded_metadata = yaml.load( @@ -2990,7 +2991,7 @@ def load_resenv_metadata(self): return templates_metada def get_or_create_research_environment_security_group( - self, resenv_metadata: ResearchEnvironmentMetadata + self, resenv_metadata: ResearchEnvironmentMetadata ): if not resenv_metadata.needs_forc_support: return None diff --git a/docker-compose.bibigrid.yml b/docker-compose.bibigrid.yml index 57bc1c64..a92cc1ba 100644 --- a/docker-compose.bibigrid.yml +++ b/docker-compose.bibigrid.yml @@ -18,7 +18,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.8 + image: redis:7.0.11 expose: - "6379" networks: @@ -41,7 +41,7 @@ services: # filebeat filebeat: - image: docker.elastic.co/beats/filebeat:8.6.2 + image: docker.elastic.co/beats/filebeat:8.7.0 env_file: - .env volumes: diff --git a/docker-compose.dev.bibigrid.yml b/docker-compose.dev.bibigrid.yml index e2372207..ac80f7f3 100644 --- a/docker-compose.dev.bibigrid.yml +++ b/docker-compose.dev.bibigrid.yml @@ -18,7 +18,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.8 + image: redis:7.0.11 expose: - "6379" networks: diff --git a/docker-compose.dev.yml b/docker-compose.dev.yml index 45b9a2c5..8e8db2ec 100644 --- a/docker-compose.dev.yml +++ b/docker-compose.dev.yml @@ -18,7 +18,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.8 + image: redis:7.0.11 expose: - "6379" networks: diff --git a/docker-compose.yml b/docker-compose.yml index 754ff789..ea2a1429 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,7 +19,7 @@ services: client_redis: container_name: client_redis - image: redis:7.0.8 + image: redis:7.0.9 expose: - "6379" networks: diff --git a/requirements.txt b/requirements.txt index ddca5c82..57275bf7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ -setuptools==67.6.0 +setuptools==67.6.1 thrift==0.16.0 -python-keystoneclient==5.0.1 +python-keystoneclient==5.1.0 openstacksdk ==1.0.1 deprecated == 1.2.13 -ansible==7.3.0 +ansible==7.4.0 Click==8.1.3 flake8==6.0.0 paramiko==3.1.0 diff --git a/requirements.yml b/requirements.yml index 3926cfed..f52480b2 100644 --- a/requirements.yml +++ b/requirements.yml @@ -7,6 +7,6 @@ roles: - name: andrewrothstein.miniconda - version: v6.1.7 + version: v6.1.8 - name: andrewrothstein.conda-env