Merge branch 'e-m-b-a:main' into pipdev

dev-tools/check_project.sh

@@ -184,7 +184,7 @@ banditer() {
     exit 1
   fi
 
-  mapfile -t PY_SCRIPTS < <(find . -type d -name migrations -prune -false -o -iname "*.py" -not -path "./.venv/*" -not -path "./emba/*")
+  mapfile -t PY_SCRIPTS < <(find . -type d -name migrations -prune -false -o -iname "*.py" -not -path "./.venv/*" -not -path "./emba/*" -not -path "./emba_logs/*")
 
   for PY_SCRIPT in "${PY_SCRIPTS[@]}"; do
     echo -e "\\n""${GREEN}""Run bandit on ${PY_SCRIPT}:""${NC}""\\n"
@@ -294,11 +294,11 @@ list_linter_exceptions(){
       SEARCH_TYPE_="sh"
       ;;
     bandit)
-      SEARCH_PAR_="nosec"
+      SEARCH_PAR_=" nosec"
       SEARCH_TYPE_="py"
       ;;
     pylint)
-      SEARCH_PAR_="pylint"
+      SEARCH_PAR_="pylint: disable"
       SEARCH_TYPE_="py"
       ;;
     djlint)
@@ -350,13 +350,14 @@ shellchecker
 list_linter_exceptions "shellcheck" "$PWD"
 dockerchecker
 jscheck
-list_linter_exceptions "jshint" "$PWD"
+list_linter_exceptions "jshint" "$PWD/embark"
 templatechecker
-list_linter_exceptions "djlint" "$PWD"
+list_linter_exceptions "djlint" "$PWD/embark"
 pycodestyle_check
 banditer
-list_linter_exceptions "bandit" "$PWD" "${PWD}/.venv"
+list_linter_exceptions "bandit" "$PWD/embark"
 pylinter
+list_linter_exceptions "pylint" "$PWD/embark"
 check_django
 yamlchecker
 copy_right_check 2024 "${PWD}" "${PWD}/emba_logs"

embark/embark/helper.py

@@ -6,8 +6,7 @@
 import os
 from pathlib import Path
 
-# from embark.settings.deploy import EMBA_ROOT
-EMBA_ROOT = os.path.join('/home/benedikt/embark', 'emba')
+from django.conf import settings
 
 
 def rnd_rgb_color():
@@ -64,32 +63,53 @@ def cleanup_charfield(charfield) -> str:
     return charfield
 
 
+def count_emba_modules(emba_dir_path):
+    s_module_cnt, p_module_cnt, q_module_cnt, l_module_cnt, f_module_cnt, d_module_cnt = 0, 0, 0, 0, 0, 0
+    for mod_file_ in os.listdir(f"{emba_dir_path}/modules"):
+        if mod_file_.startswith('S'):
+            s_module_cnt += 1
+        elif mod_file_.startswith('P'):
+            p_module_cnt += 1
+        elif mod_file_.startswith('F'):
+            f_module_cnt += 1
+        elif mod_file_.startswith('L'):
+            l_module_cnt += 1
+        elif mod_file_.startswith('Q'):
+            q_module_cnt += 1
+        elif mod_file_.startswith('D'):
+            d_module_cnt += 1
+    return s_module_cnt, p_module_cnt, q_module_cnt, l_module_cnt, f_module_cnt, d_module_cnt
+
+
 def get_version_strings():
     # gets us the currently installed version
-    if Path(EMBA_ROOT + "/external/onlinechecker").exists():
+    if Path(settings.EMBA_ROOT + "/external/onlinechecker").exists():
         # get the latest version nnumbers
-        with open(Path(EMBA_ROOT + "/external/onlinechecker/EMBA_VERSION.txt"), 'r', encoding='UTF-8') as emba_version_file:
+        with open(Path(settings.EMBA_ROOT + "/external/onlinechecker/EMBA_VERSION.txt"), 'r', encoding='UTF-8') as emba_version_file:
             stable_emba_version = emba_version_file.read().splitlines()[0]
-        with open(Path(EMBA_ROOT + "/external/onlinechecker/EMBA_CONTAINER_HASH.txt"), 'r', encoding='UTF-8') as container_version_file:
+        with open(Path(settings.EMBA_ROOT + "/external/onlinechecker/EMBA_CONTAINER_HASH.txt"), 'r', encoding='UTF-8') as container_version_file:
             container_version = container_version_file.read().splitlines()[0]
-        with open(Path(EMBA_ROOT + "/external/onlinechecker/NVD_HASH.txt"), 'r', encoding='UTF-8') as nvd_version_file:
+        with open(Path(settings.EMBA_ROOT + "/external/onlinechecker/NVD_HASH.txt"), 'r', encoding='UTF-8') as nvd_version_file:
             nvd_version = nvd_version_file.read().splitlines()[0]
-        with open(Path(EMBA_ROOT + "/external/onlinechecker/EMBA_GITHUB_HASH.txt"), 'r', encoding='UTF-8') as emba_github_version_file:
+        with open(Path(settings.EMBA_ROOT + "/external/onlinechecker/EMBA_GITHUB_HASH.txt"), 'r', encoding='UTF-8') as emba_github_version_file:
             github_emba_version = emba_github_version_file.read().splitlines()[0]
     else:
         stable_emba_version = ""
         container_version = ""
         nvd_version = ""
         github_emba_version = ""
 
-    if Path(EMBA_ROOT + "/config/VERSION.txt").exists():
-        with open(Path(EMBA_ROOT + "/config/VERSION.txt"), 'r', encoding='UTF-8') as emba_version_file:
+    if Path(settings.EMBA_ROOT + "/config/VERSION.txt").exists():
+        with open(Path(settings.EMBA_ROOT + "/config/VERSION.txt"), 'r', encoding='UTF-8') as emba_version_file:
             emba_version = emba_version_file.read().splitlines()[0]
     else:
         emba_version = ""
 
-    if Path("./VERSION.txt").exists():
-        with open(Path("./VERSION.txt"), 'r', encoding='UTF-8') as embark_version_file:
+    if Path(f"{settings.BASE_DIR}/VERSION.txt").exists():
+        with open(Path(f"{settings.BASE_DIR}/VERSION.txt"), 'r', encoding='UTF-8') as embark_version_file:
+            embark_version = embark_version_file.read().splitlines()[0]
+    elif Path(f"{settings.BASE_DIR.parent}/VERSION.txt").exists():
+        with open(Path(f"{settings.BASE_DIR.parent}/VERSION.txt"), 'r', encoding='UTF-8') as embark_version_file:
             embark_version = embark_version_file.read().splitlines()[0]
     else:
         embark_version = ""
@@ -101,4 +121,4 @@ def get_version_strings():
     TEST_STRING = 'Linux / v2.6.33.2'
     print(cleanup_charfield(TEST_STRING))
 
-    print(get_version_strings())
+    print(count_emba_modules(emba_dir_path="/home/cylox/embark/emba"))

embark/embark/logreader.py

@@ -112,8 +112,8 @@ def phase_identify(status_message):
         failed_pattern = "EMBA failed in docker mode!"
 
         # calculate percentage
-        max_module = -1
-        phase_nmbr = -1
+        max_module = -2
+        phase_nmbr = -2
         if re.search(pattern=re.escape(pre_checker_phase_pattern), string=status_message["phase"]):
             max_module = EMBA_P_MOD_CNT
             phase_nmbr = EMBA_P_PHASE
@@ -130,8 +130,11 @@ def phase_identify(status_message):
             max_module = 0
             phase_nmbr = EMBA_PHASE_CNT
         elif re.search(pattern=re.escape(failed_pattern), string=status_message["phase"]):
-            max_module = -2
+            max_module = -1
             phase_nmbr = EMBA_PHASE_CNT
+        else:
+            logger.info("Undefined pattern in logreader %s ", status_message["phase"])
+            logger.info("Not updating status percentage")
         return max_module, phase_nmbr
 
     # update our dict whenever a new module is being processed
@@ -144,14 +147,20 @@ def update_status(self, stream_item_list):
         elif max_module > 0:
             self.module_cnt += 1
             self.module_cnt = self.module_cnt % max_module  # make sure it's in range
-            percentage = phase_nmbr * (100 / EMBA_PHASE_CNT) + ((100 / EMBA_PHASE_CNT) / max_module) * self.module_cnt   # increments: F=6.25, S=0.65, L=3.57, P=1.25
+            percentage = phase_nmbr * (100 / EMBA_PHASE_CNT) + ((100 / EMBA_PHASE_CNT) / max_module) * self.module_cnt
+        elif max_module == -1:
+            percentage = 100
+            self.finish = True
+            logger.error("EMBA failed with  %s ", self.status_msg)
         else:
-            logger.debug("Undefined state in logreader %s ", self.status_msg)
+            logger.error("Undefined state in logreader %s ", self.status_msg)
+            percentage = self.status_msg["percentage"]  # stays the same
 
         logger.debug("Status is %d, in phase %d, with modules %d", percentage, phase_nmbr, max_module)
 
         # set attributes of current message
         self.status_msg["module"] = stream_item_list[0]
+
         # ignore all Q-modules for percentage calc
         if not re.match(".*Q[0-9][0-9]", stream_item_list[0]):
             self.status_msg["percentage"] = percentage

embark/embark/settings/deploy.py

@@ -12,7 +12,7 @@
 
 from dotenv import load_dotenv
 
-from embark.helper import get_version_strings
+from embark.helper import count_emba_modules, get_version_strings
 
 # load .env file
 load_dotenv()
@@ -307,30 +307,9 @@
 SECURE_HSTS_SECONDS = 0
 SECURE_SSL_REDIRECT = False
 
-
-def count_emba_modules(emba_dir_path):
-    s_module_cnt, p_module_cnt, q_module_cnt, l_module_cnt, f_module_cnt = 0, 0, 0, 0, 0
-    for mod_file_ in os.listdir(f"{emba_dir_path}/modules"):
-        if mod_file_.startswith('S'):
-            s_module_cnt += 1
-        elif mod_file_.startswith('P'):
-            p_module_cnt += 1
-        elif mod_file_.startswith('F'):
-            f_module_cnt += 1
-        elif mod_file_.startswith('L'):
-            l_module_cnt += 1
-        elif mod_file_.startswith('Q'):
-            q_module_cnt += 1
-    return s_module_cnt, p_module_cnt, f_module_cnt, l_module_cnt, q_module_cnt
-
-
 try:
-    EMBA_S_MOD_CNT, EMBA_P_MOD_CNT, EMBA_F_MOD_CNT, EMBA_L_MOD_CNT, EMBA_Q_MOD_CNT = count_emba_modules(EMBA_ROOT)
+    EMBA_S_MOD_CNT, EMBA_P_MOD_CNT, EMBA_Q_MOD_CNT, EMBA_L_MOD_CNT, EMBA_F_MOD_CNT, EMBA_D_MOD_CNT = count_emba_modules(EMBA_ROOT)
 except FileNotFoundError as file_error:
-    print("[Warning] Installation is missing the EMBA submodule")
-    EMBA_S_MOD_CNT = 44
-    EMBA_P_MOD_CNT = 18
-    EMBA_F_MOD_CNT = 4
-    EMBA_L_MOD_CNT = 8
+    EMBA_S_MOD_CNT, EMBA_P_MOD_CNT, EMBA_Q_MOD_CNT, EMBA_L_MOD_CNT, EMBA_F_MOD_CNT, EMBA_D_MOD_CNT = 46, 20, 1, 10, 6, 3
 
 VERSION = get_version_strings()

embark/embark/settings/dev.py

@@ -8,7 +8,7 @@
 
 from dotenv import load_dotenv
 
-from embark.helper import get_version_strings
+from embark.helper import count_emba_modules, get_version_strings
 
 # Build paths inside the project like this: BASE_DIR / 'subdir'.
 BASE_DIR = Path(__file__).resolve().parent.parent.parent
@@ -271,29 +271,9 @@
 }
 TEMP_DIR = Path("/tmp/")
 
-
-def count_emba_modules(emba_dir_path):
-    s_module_cnt, p_module_cnt, q_module_cnt, l_module_cnt, f_module_cnt = 0, 0, 0, 0, 0
-    for mod_file_ in os.listdir(f"{emba_dir_path}/modules"):
-        if mod_file_.startswith('S'):
-            s_module_cnt += 1
-        elif mod_file_.startswith('P'):
-            p_module_cnt += 1
-        elif mod_file_.startswith('F'):
-            f_module_cnt += 1
-        elif mod_file_.startswith('L'):
-            l_module_cnt += 1
-        elif mod_file_.startswith('Q'):
-            q_module_cnt += 1
-    return s_module_cnt, p_module_cnt, f_module_cnt, l_module_cnt, q_module_cnt
-
-
 try:
-    EMBA_S_MOD_CNT, EMBA_P_MOD_CNT, EMBA_F_MOD_CNT, EMBA_L_MOD_CNT, EMBA_Q_MOD_CNT = count_emba_modules(EMBA_ROOT)
+    EMBA_S_MOD_CNT, EMBA_P_MOD_CNT, EMBA_Q_MOD_CNT, EMBA_L_MOD_CNT, EMBA_F_MOD_CNT, EMBA_D_MOD_CNT = count_emba_modules(EMBA_ROOT)
 except FileNotFoundError as file_error:
-    EMBA_S_MOD_CNT = 44
-    EMBA_P_MOD_CNT = 18
-    EMBA_F_MOD_CNT = 4
-    EMBA_L_MOD_CNT = 8
+    EMBA_S_MOD_CNT, EMBA_P_MOD_CNT, EMBA_Q_MOD_CNT, EMBA_L_MOD_CNT, EMBA_F_MOD_CNT, EMBA_D_MOD_CNT = 46, 20, 1, 10, 6, 3
 
 VERSION = get_version_strings()

embark/reporter/urls.py

@@ -10,14 +10,12 @@
 
 # view routing
 urlpatterns = [
-    path(settings.EMBA_LOG_URL + '<uuid:analysis_id>/emba_logs/html-report/style/<str:img_file>', views.html_report_resource, name='embark-html-report-resource'),
-    path(settings.EMBA_LOG_URL + '<uuid:analysis_id>/emba_logs/html-report/<str:html_file>', views.html_report, name='embark-html-report'),
-    path(settings.EMBA_LOG_URL + '<uuid:analysis_id>/emba_logs/html-report/<str:html_path>/<str:html_file>', views.html_report_path, name='embark-html-report-path'),
-
+    # TODO get rid of the emba log paths
+    path(settings.EMBA_LOG_URL + '<uuid:analysis_id>/<str:html_file>', views.html_report, name='embark-html-report-index'),
+    path(settings.EMBA_LOG_URL + '<uuid:analysis_id>/style/<str:img_file>', views.html_report_resource, name='embark-html-report-resource'),
+    path(settings.EMBA_LOG_URL + '<uuid:analysis_id>/<path:html_path>/<str:file>', views.html_report_path, name='embark-html-report-path'),
     path('get_load/', views.get_load, name='embark-get-load'),
     path('get_individual_report/<uuid:analysis_id>/', views.get_individual_report, name='embark-get-individual-report'),
     path('get_accumulated_reports/', views.get_accumulated_reports, name='embark-get-accumulated-reports'),
-
-    path(settings.EMBA_LOG_URL + '<uuid:analysis_id>/emba_logs/html-report/<path:html_path>/<str:download_file>/', views.html_report_download, name='embark-html-report-download'),
     path('download_zipped/<uuid:analysis_id>/', views.download_zipped, name='embark-download'),
 ]