Merge pull request #411 from deNBI/feat/delete_matching_upstream_urls

Feat/delete matching upstream urls

FastapiOpenRestyConfigurator/app/main/model/serializers.py

@@ -125,6 +125,11 @@ class BackendOut(BackendBase):
         description="Protected reverse-proxy path which leads to specific backend.",
         example="myRstudio_103"
     )
+    file_path:str=Field(
+        None,
+        title="File path of Backend",
+        description=" Local File path of the backend",
+    )
 
 
 class BackendTemp(BackendIn, BackendOut):

FastapiOpenRestyConfigurator/app/main/service/backend.py

@@ -5,7 +5,7 @@
 import os
 import re
 from random import randint
-from typing import List
+from typing import List, Dict
 
 from werkzeug.exceptions import NotFound, InternalServerError
 
@@ -19,9 +19,10 @@
 
 file_regex = r"(\d*)%([a-z0-9\-\@.]*?)%([^%]*)%([^%]*)%([^%]*)\.conf"
 
+
 async def random_with_n_digits(n):
-    range_start = 10**(n-1)
-    range_end = (10**n)-1
+    range_start = 10 ** (n - 1)
+    range_end = (10 ** n) - 1
     return randint(range_start, range_end)
 
 
@@ -40,16 +41,43 @@ async def get_backends() -> List[BackendOut]:
             logger.warning("Found a backend file with wrong naming, skipping it: " + str(file))
             continue
         backend: BackendOut = BackendOut(
-            id=match.group(1),
+                id=match.group(1),
             owner=match.group(2),
             location_url=match.group(3),
             template=match.group(4),
-            template_version=match.group(5)
+            template_version=match.group(5),
+            file_path=os.path.join(settings.FORC_BACKEND_PATH, file)
         )
         valid_backends.append(backend)
     return valid_backends
 
 
+async def get_backends_upstream_urls() -> Dict[str, List[BackendOut]]:
+    valid_backends: List[BackendOut] = await get_backends()
+    upstream_urls = {}
+
+    for backend in valid_backends:
+        upstream_url = extract_proxy_pass(backend.file_path)
+        if upstream_url:
+            if upstream_url not in upstream_urls:
+                upstream_urls[upstream_url] = []
+            upstream_urls[upstream_url].append(backend)
+
+    return upstream_urls
+
+
+def extract_proxy_pass(file_path):
+    with open(file_path, 'r') as file:
+        content = file.read()
+
+    match = re.search(r'proxy_pass\s+(http[^\s;]+);', content)
+
+    if match:
+        return match.group(1)
+    else:
+        return None
+
+
 async def generate_suffix_number(user_key_url):
     current_backends: List[BackendOut] = await get_backends()
     same_name_backend_ids = []
@@ -82,16 +110,12 @@ async def create_backend(payload: BackendIn):
 
     payload.location_url = f"{payload.user_key_url}_{suffix_number}"
 
-    # check for duplicates in filesystem
-    backend_path_files = os.listdir(settings.FORC_BACKEND_PATH)
-    for file in backend_path_files:
-        match = re.fullmatch(file_regex, file)
-        if not match:
-            logger.warning("Found a backend file with wrong naming, skipping it: " + str(file))
-            continue
-        if match.group(1) == payload.id or match.group(3) == payload.location_url:
-            logger.error("Tried to create duplicate backend: " + payload.id + " and " + payload.location_url)
-            raise InternalServerError("Server tried to create duplicate backend.")
+    # check for duplicated in ip and port:
+    upstream_urls: Dict[str, List[BackendOut]] = await get_backends_upstream_urls()
+    matching_urls_backends: List[BackendOut] = upstream_urls.get(payload.upstream_url, [])
+    for backend in matching_urls_backends:
+        logger.info(f"Deleting existing Backend with same Upstream Url - {payload.upstream_url}")
+        await delete_backend(backend.id)
 
     # create backend file in filesystem
     filename = f"{payload.id}%{payload.owner}%{payload.location_url}%{payload.template}%{payload.template_version}.conf"

ansible/roles/forc_api/tasks/025-backup.yml

@@ -34,24 +34,26 @@
     src: "{{ role_path }}/files"
     dest: "/persistent"
 
-- docker_container:
-    name: "backup-forc"
-    image: quay.io/denbicloud/cron-backup:0.13.5
-
-    state: started
-    restart: yes
-    volumes:
-      - "{{FORC_BACKEND_PATH}}:/backends"
-      - "{{FORC_TEMPLATE_PATH}}:/templates"
-      - "/persistent/files/install-packages.sh:/install-packages.sh"
-      - "/persistent/files/backup.sh:/etc/cronscripts/backup.sh"
-      - "/persistent/files/backup-cron:/etc/crontabs/dockercron/backup-cron"
-      - "{{FORC_BACKUP_HOST_PATH}}:/etc/backup"
-    env:
-      BACKUP_ROTATION_ENABLED={{FORC_BACKUP_ROTATION_ENABLED}}
-      BACKUP_ROTATION_MAX_SIZE={{FORC_BACKUP_ROTATION_MAX_SIZE}}
-      BACKUP_ROTATION_CUT_SIZE={{FORC_BACKUP_ROTATION_CUT_SIZE}}
-      BACKUP_ROTATION_SIZE_TYP={{FORC_BACKUP_ROTATION_SIZE_TYP}}
+- name: Run the backup-forc container
+  shell: |
+    docker run -d \
+      --name backup-forc \
+      --restart always \
+      -v "{{ FORC_BACKEND_PATH }}:/backends" \
+      -v "{{ FORC_TEMPLATE_PATH }}:/templates" \
+      -v "/persistent/files/install-packages.sh:/install-packages.sh" \
+      -v "/persistent/files/backup.sh:/etc/cronscripts/backup.sh" \
+      -v "/persistent/files/backup-cron:/etc/crontabs/dockercron/backup-cron" \
+      -v "{{ FORC_BACKUP_HOST_PATH }}:/etc/backup" \
+      -e BACKUP_ROTATION_ENABLED={{ FORC_BACKUP_ROTATION_ENABLED }} \
+      -e BACKUP_ROTATION_MAX_SIZE={{ FORC_BACKUP_ROTATION_MAX_SIZE }} \
+      -e BACKUP_ROTATION_CUT_SIZE={{ FORC_BACKUP_ROTATION_CUT_SIZE }} \
+      -e BACKUP_ROTATION_SIZE_TYP={{ FORC_BACKUP_ROTATION_SIZE_TYP }} \
+      quay.io/denbicloud/cron-backup:0.13.5
+  args:
+    executable: /bin/bash
+
+
 
 
 

ansible/roles/forc_api/tasks/main.yml

@@ -1,23 +1,34 @@
 ---
 # tasks file for forc_api
 
-- include: "000-preparation.yml"
-  tags: ['forc_preparation']
+- name: Preparation tasks
+  import_tasks: "000-preparation.yml"
+  tags:
+    - forc_preparation
 
-- include: "005-packages.yml"
-  tags: ['forc_packages']
-
-- include: "010-config.yml"
-  tags: ['forc_config']
+- name: Package installation tasks
+  import_tasks: "005-packages.yml"
+  tags:
+    - forc_packages
 
-- include: "015-forc.yml"
-  tags: ['forc_forc']
+- name: Configuration tasks
+  import_tasks: "010-config.yml"
+  tags:
+    - forc_config
 
-- include: "020-openresty.yml"
-  tags: ['forc_openresty']
+- name: FORC tasks
+  import_tasks: "015-forc.yml"
+  tags:
+    - forc_forc
+
+- name: OpenResty installation tasks
+  import_tasks: "020-openresty.yml"
   when: INSTALL_OPENRESTY
+  tags:
+    - forc_openresty
 
-- include: "025-backup.yml"
-  tags: ["forc-backup"]
+- name: Backup tasks
+  import_tasks: "025-backup.yml"
   when: FORC_BACKUP_ENABLED
-
+  tags:
+    - forc_backup