From 79126adb86dcda38449261ddd79f0d9e8973fb7c Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Wed, 22 Nov 2023 10:18:35 +0000 Subject: [PATCH 01/17] ch placeholder top button --- embark/templates/dashboard/reportDashboard.html | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/embark/templates/dashboard/reportDashboard.html b/embark/templates/dashboard/reportDashboard.html index 2224d8bb..3ccb6ba9 100644 --- a/embark/templates/dashboard/reportDashboard.html +++ b/embark/templates/dashboard/reportDashboard.html @@ -52,9 +52,11 @@ {% if firmware.end_date is not None %} {{ firmware.end_date}} {% else %} - +
+ +
{% endif %} From 3a88e8bc626ed727cd5f5f68373aa18e468848e0 Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 08:27:18 +0000 Subject: [PATCH 02/17] fix missing apt update --- .github/workflows/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e9531f5f..e7b3ee25 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -21,6 +21,7 @@ jobs: uses: actions/checkout@v3 - name: Install dependencies for linting run: | + sudo apt-get update -y sudo apt-get install -y python3-dev npm pycodestyle python3-pylint-django bandit sudo npm install -g jshint - name: Install Python, pipenv and Pipfile packages From 0fb6e16fca2efd4f1f28529b318dea5fa25999ae Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 09:36:34 +0000 Subject: [PATCH 03/17] no naive time values --- embark/embark/logreader.py | 4 ++-- embark/uploader/boundedexecutor.py | 6 +++--- embark/uploader/management/commands/runapscheduler.py | 5 ++--- embark/users/views.py | 6 +++--- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/embark/embark/logreader.py b/embark/embark/logreader.py index 4d204051..53a20b6b 100644 --- a/embark/embark/logreader.py +++ b/embark/embark/logreader.py @@ -1,7 +1,7 @@ # pylint: disable=W0602 # ignores no-assignment error since there is one! import builtins -import datetime +from django.utils import timezone import difflib import pathlib import re @@ -75,7 +75,7 @@ def save_status(self): logger.debug("Appending status with message: %s", self.status_msg) # append message to the json-field structure of the analysis self.analysis.status["percentage"] = self.status_msg["percentage"] - self.analysis.status["last_update"] = str(datetime.datetime.now()) + self.analysis.status["last_update"] = str(timezone.now()) # append modules and phase list if self.status_msg["module"] != self.analysis.status["last_module"]: self.analysis.status["last_module"] = self.status_msg["module"] diff --git a/embark/uploader/boundedexecutor.py b/embark/uploader/boundedexecutor.py index 387395c6..89912b1e 100644 --- a/embark/uploader/boundedexecutor.py +++ b/embark/uploader/boundedexecutor.py @@ -11,7 +11,7 @@ from threading import BoundedSemaphore from django.dispatch import receiver -from django.utils.datetime_safe import datetime +from django.utils import timezone from django.conf import settings from uploader import finish_execution @@ -116,8 +116,8 @@ def run_emba_cmd(cls, cmd, analysis_id=None, active_analyzer_dir=None): # finalize db entry if analysis: - analysis.end_date = datetime.now() - analysis.scan_time = datetime.now() - analysis.start_date + analysis.end_date = timezone.now() + analysis.scan_time = timezone.now() - analysis.start_date analysis.duration = str(analysis.scan_time) analysis.finished = True analysis.failed = exit_fail diff --git a/embark/uploader/management/commands/runapscheduler.py b/embark/uploader/management/commands/runapscheduler.py index e5905ab1..70287761 100644 --- a/embark/uploader/management/commands/runapscheduler.py +++ b/embark/uploader/management/commands/runapscheduler.py @@ -1,4 +1,3 @@ -import datetime as dtime import logging import psutil @@ -6,10 +5,10 @@ from apscheduler.triggers.cron import CronTrigger from django.conf import settings from django.core.management.base import BaseCommand -from django.utils.datetime_safe import datetime from django_apscheduler.jobstores import DjangoJobStore from django_apscheduler.models import DjangoJobExecution from django_apscheduler import util +from django.utils import timezone from uploader.models import ResourceTimestamp @@ -54,7 +53,7 @@ def delete_old_job_executions(max_age=1_209_600): DjangoJobExecution.objects.delete_old_job_executions(max_age) # delete entries in db older than 2 weeks - time_delta = datetime.now() - dtime.timedelta(seconds=max_age) + time_delta = timezone.now() - timezone.timedelta(seconds=max_age) ResourceTimestamp.objects.all().filter(timestamp__lt=time_delta).delete() diff --git a/embark/users/views.py b/embark/users/views.py index 17112f02..6795f68c 100644 --- a/embark/users/views.py +++ b/embark/users/views.py @@ -1,7 +1,6 @@ # pylint: disable=R1705 import builtins import logging -from datetime import datetime from django.contrib.auth.decorators import login_required from django.shortcuts import render, redirect @@ -11,6 +10,7 @@ from django.contrib.auth import authenticate, login, logout, get_user from django.contrib import messages from django.conf import settings +from django.utils import timezone from users.models import User @@ -162,8 +162,8 @@ def acc_delete(request): if request.method == "POST": logger.debug('disabling account') user = get_user(request) - logger.debug(' %s Account: %s disabled', datetime.now().strftime("%H:%M:%S"), user) - user.username = user.get_username() + '_disactivated_' + datetime.now().strftime( + logger.debug(' %s Account: %s disabled', timezone.now().strftime("%H:%M:%S"), user) + user.username = user.get_username() + '_disactivated_' + timezone.now().strftime( "%H:%M:%S") # workaround for not duplicating entry users_user.username user.is_active = False user.save() From 0f7326db4dfa80fdba52ae0b731a3c9877696f3f Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 09:46:11 +0000 Subject: [PATCH 04/17] switch to timezone --- embark/porter/models.py | 4 ++-- embark/tracker/views.py | 4 ++-- embark/uploader/models.py | 17 +++++++---------- 3 files changed, 11 insertions(+), 14 deletions(-) diff --git a/embark/porter/models.py b/embark/porter/models.py index d36295ce..425399eb 100644 --- a/embark/porter/models.py +++ b/embark/porter/models.py @@ -5,7 +5,7 @@ from django.conf import settings from django.db.models.signals import pre_delete from django.dispatch import receiver -from django.utils.datetime_safe import datetime +from django.utils import timezone from django.db import models from users.models import User as Userclass @@ -20,7 +20,7 @@ class LogZipFile """ id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True) - upload_date = models.DateTimeField(default=datetime.now, blank=True) + upload_date = models.DateTimeField(default=timezone.now, blank=True) user = models.ForeignKey(Userclass, on_delete=models.SET_NULL, related_name='Import_zip_Upload_User', null=True, blank=True) def get_storage_path(self, filename): diff --git a/embark/tracker/views.py b/embark/tracker/views.py index 759f33b1..5226d023 100644 --- a/embark/tracker/views.py +++ b/embark/tracker/views.py @@ -1,4 +1,3 @@ -from datetime import datetime, timedelta import logging from django.conf import settings @@ -8,6 +7,7 @@ from django.views.decorators.http import require_http_methods from django.shortcuts import redirect from django.contrib import messages +from django.utils import timezone from django_tables2 import RequestConfig @@ -44,7 +44,7 @@ def tracker(request): return redirect('embark-uploader-home') logger.error("invalid date form") return redirect('..') - date = datetime.today() - timedelta(days=7) + date = timezone.localdate() - timezone.timedelta(days=7) vendor_list = Vendor.objects.all() if vendor_list.count() != 0: label_list = [] diff --git a/embark/uploader/models.py b/embark/uploader/models.py index 4e1bb8dc..0a3f56b9 100644 --- a/embark/uploader/models.py +++ b/embark/uploader/models.py @@ -10,14 +10,11 @@ from django import forms from django.db.models.signals import pre_delete from django.dispatch import receiver -from django.utils.datetime_safe import datetime -from porter.models import LogZipFile - -# from hashid_field import HashidAutoField +from django.utils import timezone +from porter.models import LogZipFile from users.models import User as Userclass - logger = logging.getLogger(__name__) @@ -128,7 +125,7 @@ class FirmwareFile id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=True) is_archive = models.BooleanField(default=False, blank=True) - upload_date = models.DateTimeField(default=datetime.now, blank=True) + upload_date = models.DateTimeField(default=timezone.now, blank=True) user = models.ForeignKey(Userclass, on_delete=models.SET_NULL, related_name='Fw_Upload_User', null=True, blank=True) def get_storage_path(self, filename): @@ -193,7 +190,7 @@ class Label label_name = models.CharField( help_text='label name', verbose_name="label name", max_length=MAX_LENGTH, blank=True, unique=True) - label_date = models.DateTimeField(default=datetime.now, blank=True) + label_date = models.DateTimeField(default=timezone.now, blank=True) class Meta: ordering = ['label_name'] @@ -216,7 +213,7 @@ class Device device_name = models.CharField(help_text='Device name', verbose_name="Device name", max_length=MAX_LENGTH, blank=True) device_vendor = models.ForeignKey(Vendor, on_delete=models.SET_NULL, null=True) device_label = models.ForeignKey(Label, on_delete=models.SET_NULL, null=True, help_text='label/tag', related_query_name='label', editable=True, blank=True) # TODO make many to many field - device_date = models.DateTimeField(default=datetime.now, blank=True) + device_date = models.DateTimeField(default=timezone.now, blank=True) device_user = models.ForeignKey(Userclass, on_delete=models.SET_NULL, related_name='Device_User', null=True) # TODO change acces control to usergroup?? visible = models.BooleanField(editable=True, default=True) @@ -314,7 +311,7 @@ class Firmware # embark meta data path_to_logs = models.FilePathField(default="/", blank=True) log_size = models.PositiveBigIntegerField(default=0, blank=True) - start_date = models.DateTimeField(default=datetime.now, blank=True) + start_date = models.DateTimeField(default=timezone.now, blank=True) end_date = models.DateTimeField(default=None, null=True) scan_time = models.DurationField(default=None, null=True) duration = models.CharField(blank=True, null=True, max_length=100, help_text='') @@ -420,6 +417,6 @@ class ResourceTimestamp Model to store zipped or bin firmware file and upload date """ - timestamp = models.DateTimeField(default=datetime.now) + timestamp = models.DateTimeField(default=timezone.now) cpu_percentage = models.FloatField(default=0.0) memory_percentage = models.FloatField(default=0.0) From 6749b7edc5b50de9449d54d10ba90d46d3c9908b Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 09:46:58 +0000 Subject: [PATCH 05/17] add timezone set option via env-var --- embark/embark/settings/deploy.py | 2 +- embark/embark/settings/dev.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/embark/embark/settings/deploy.py b/embark/embark/settings/deploy.py index df432528..35a1078e 100644 --- a/embark/embark/settings/deploy.py +++ b/embark/embark/settings/deploy.py @@ -224,7 +224,7 @@ LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC') USE_I18N = True diff --git a/embark/embark/settings/dev.py b/embark/embark/settings/dev.py index 63dbe03c..cfe99852 100644 --- a/embark/embark/settings/dev.py +++ b/embark/embark/settings/dev.py @@ -190,7 +190,7 @@ LANGUAGE_CODE = 'en-us' -TIME_ZONE = 'UTC' +TIME_ZONE = os.environ.get('TIME_ZONE', 'UTC') USE_I18N = True From d9a0ed2800dac7dc0e5a353dec2d472e12ebb93d Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 11:49:07 +0000 Subject: [PATCH 06/17] fix import oders --- embark/embark/logreader.py | 2 +- embark/uploader/management/commands/runapscheduler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/embark/embark/logreader.py b/embark/embark/logreader.py index 53a20b6b..3bc64389 100644 --- a/embark/embark/logreader.py +++ b/embark/embark/logreader.py @@ -1,7 +1,6 @@ # pylint: disable=W0602 # ignores no-assignment error since there is one! import builtins -from django.utils import timezone import difflib import pathlib import re @@ -15,6 +14,7 @@ from channels.layers import get_channel_layer from django.conf import settings +from django.utils import timezone from uploader.models import FirmwareAnalysis diff --git a/embark/uploader/management/commands/runapscheduler.py b/embark/uploader/management/commands/runapscheduler.py index 70287761..88f5cd73 100644 --- a/embark/uploader/management/commands/runapscheduler.py +++ b/embark/uploader/management/commands/runapscheduler.py @@ -5,10 +5,10 @@ from apscheduler.triggers.cron import CronTrigger from django.conf import settings from django.core.management.base import BaseCommand +from django.utils import timezone from django_apscheduler.jobstores import DjangoJobStore from django_apscheduler.models import DjangoJobExecution from django_apscheduler import util -from django.utils import timezone from uploader.models import ResourceTimestamp From 84841e14335599e777352ef02f6df5ea7f5aed8a Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 12:00:16 +0000 Subject: [PATCH 07/17] add back button --- embark/templates/dashboard/logViewer.html | 3 +++ 1 file changed, 3 insertions(+) diff --git a/embark/templates/dashboard/logViewer.html b/embark/templates/dashboard/logViewer.html index db4f555f..334f50cf 100644 --- a/embark/templates/dashboard/logViewer.html +++ b/embark/templates/dashboard/logViewer.html @@ -10,6 +10,9 @@
+
+ +
From b75608d1db8eab31caf413f6f03eff61833bee45 Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 12:00:34 +0000 Subject: [PATCH 08/17] add message on redirect --- embark/dashboard/views.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/embark/dashboard/views.py b/embark/dashboard/views.py index f5439d3b..c82e11a2 100644 --- a/embark/dashboard/views.py +++ b/embark/dashboard/views.py @@ -6,7 +6,7 @@ from django.conf import settings from django.shortcuts import render -from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseRedirect, HttpResponseServerError +from django.http import HttpResponse, HttpResponseBadRequest, HttpResponseForbidden, HttpResponseServerError from django.contrib.auth.decorators import login_required from django.views.decorators.http import require_http_methods from django.contrib import messages @@ -29,7 +29,8 @@ def main_dashboard(request): if request.user.is_authenticated: if FirmwareAnalysis.objects.filter(finished=True, failed=False).count() > 0 and Result.objects.all().count() > 0: return render(request, 'dashboard/mainDashboard.html', {'nav_switch': True, 'username': request.user.username}) - return HttpResponseRedirect('../../uploader/') + messages.info(request,"There are no Results to display yet, redirecting to uploader") + return redirect('embark-uploader-home') return HttpResponseForbidden From 52fed898bb03aed92f2512f3565b89899ea60458 Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 12:01:40 +0000 Subject: [PATCH 09/17] change redirect message --- embark/dashboard/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embark/dashboard/views.py b/embark/dashboard/views.py index c82e11a2..8eac43e9 100644 --- a/embark/dashboard/views.py +++ b/embark/dashboard/views.py @@ -29,7 +29,7 @@ def main_dashboard(request): if request.user.is_authenticated: if FirmwareAnalysis.objects.filter(finished=True, failed=False).count() > 0 and Result.objects.all().count() > 0: return render(request, 'dashboard/mainDashboard.html', {'nav_switch': True, 'username': request.user.username}) - messages.info(request,"There are no Results to display yet, redirecting to uploader") + messages.info(request,"Redirected - There are no Results to display yet") return redirect('embark-uploader-home') return HttpResponseForbidden From 910885bc5d36f2f6b8a662493ffc36a7286966d8 Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 12:19:32 +0000 Subject: [PATCH 10/17] ' ' --- embark/dashboard/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/embark/dashboard/views.py b/embark/dashboard/views.py index 8eac43e9..a5037ff8 100644 --- a/embark/dashboard/views.py +++ b/embark/dashboard/views.py @@ -29,7 +29,7 @@ def main_dashboard(request): if request.user.is_authenticated: if FirmwareAnalysis.objects.filter(finished=True, failed=False).count() > 0 and Result.objects.all().count() > 0: return render(request, 'dashboard/mainDashboard.html', {'nav_switch': True, 'username': request.user.username}) - messages.info(request,"Redirected - There are no Results to display yet") + messages.info(request, "Redirected - There are no Results to display yet") return redirect('embark-uploader-home') return HttpResponseForbidden From a2526510c60cce8660a0e111670232dfc5113311 Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 12:30:05 +0000 Subject: [PATCH 11/17] set analysis failed if stop_analysis gets error --- embark/dashboard/views.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/embark/dashboard/views.py b/embark/dashboard/views.py index a5037ff8..400b3619 100644 --- a/embark/dashboard/views.py +++ b/embark/dashboard/views.py @@ -61,6 +61,9 @@ def stop_analysis(request): return render(request, 'dashboard/serviceDashboard.html', {'username': request.user.username, 'form': form, 'success_message': True, 'message': "Stopped successfully"}) except builtins.Exception as error: logger.error("Error %s", error) + analysis_object_ = FirmwareAnalysis.objects.get(id=analysis.id) + analysis_object_.failed = True + analysis_object_.save(update_fields=["failed"]) return HttpResponseServerError("Failed to stop process, please handle manually: PID=" + str(pid)) return HttpResponseBadRequest("invalid form") From 5d8e3e83df4f357a21d5e2b75c82e7229c78ee76 Mon Sep 17 00:00:00 2001 From: Benedikt Kuehne Date: Thu, 23 Nov 2023 12:46:39 +0000 Subject: [PATCH 12/17] make failed deletable even on error --- embark/dashboard/views.py | 2 +- .../templates/dashboard/reportDashboard.html | 54 +++++++++++-------- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/embark/dashboard/views.py b/embark/dashboard/views.py index 400b3619..5f947a84 100644 --- a/embark/dashboard/views.py +++ b/embark/dashboard/views.py @@ -64,7 +64,7 @@ def stop_analysis(request): analysis_object_ = FirmwareAnalysis.objects.get(id=analysis.id) analysis_object_.failed = True analysis_object_.save(update_fields=["failed"]) - return HttpResponseServerError("Failed to stop process, please handle manually: PID=" + str(pid)) + return HttpResponseServerError("Failed to stop process, but set its status to failed. Please handle EMBA process manually: PID=" + str(pid)) return HttpResponseBadRequest("invalid form") diff --git a/embark/templates/dashboard/reportDashboard.html b/embark/templates/dashboard/reportDashboard.html index 3ccb6ba9..32011cea 100644 --- a/embark/templates/dashboard/reportDashboard.html +++ b/embark/templates/dashboard/reportDashboard.html @@ -63,6 +63,14 @@
{% if firmware.finished is False %} + {% if firmware.failed is True %} + + + {% endif %}
- -
-

NewImport functionality

+

NewNew Buttons everywhere