From 76af700a4f4c621696f88a5cdf28b352f9eb1b01 Mon Sep 17 00:00:00 2001 From: Ptitloup Date: Mon, 3 Jul 2023 15:48:24 +0200 Subject: [PATCH] use file to temporarily store webvtt - rename pod-front docker to pod-back (front is nginx) --- docker-compose-full-dev-with-volumes.yml | 10 +++++----- .../{pod-front => pod-back}/Dockerfile | 8 ++++---- .../my-entrypoint-back.sh} | 4 ++-- .../pod-encode/my-entrypoint-encode.sh | 4 ++-- .../pod-transcript/my-entrypoint-transcript.sh | 4 ++-- .../importing_transcript_tasks.py | 13 ++++++++----- .../transcripting_tasks.py | 16 ++++++++++++++-- 7 files changed, 37 insertions(+), 22 deletions(-) rename dockerfile-dev-with-volumes/{pod-front => pod-back}/Dockerfile (83%) rename dockerfile-dev-with-volumes/{pod-front/my-entrypoint.sh => pod-back/my-entrypoint-back.sh} (93%) diff --git a/docker-compose-full-dev-with-volumes.yml b/docker-compose-full-dev-with-volumes.yml index ebdf304b3c..0a57065f03 100755 --- a/docker-compose-full-dev-with-volumes.yml +++ b/docker-compose-full-dev-with-volumes.yml @@ -7,11 +7,11 @@ x-elasticsearch-volumes: &elasticsearch-volumes version: '3.7' services: - pod-front: - container_name: pod-front-with-volumes + pod-back: + container_name: pod-back-with-volumes build: context: . - dockerfile: dockerfile-dev-with-volumes/pod-front/Dockerfile + dockerfile: dockerfile-dev-with-volumes/pod-back/Dockerfile depends_on: - elasticsearch - redis @@ -27,7 +27,7 @@ services: context: . dockerfile: dockerfile-dev-with-volumes/pod-encode/Dockerfile depends_on: - - pod-front + - pod-back env_file: - ./.env.dev volumes: *pod-volumes @@ -38,7 +38,7 @@ services: context: . dockerfile: dockerfile-dev-with-volumes/pod-transcript/Dockerfile depends_on: - - pod-front + - pod-back env_file: - ./.env.dev volumes: *pod-volumes diff --git a/dockerfile-dev-with-volumes/pod-front/Dockerfile b/dockerfile-dev-with-volumes/pod-back/Dockerfile similarity index 83% rename from dockerfile-dev-with-volumes/pod-front/Dockerfile rename to dockerfile-dev-with-volumes/pod-back/Dockerfile index 3ed5a69987..0e1d0792d7 100755 --- a/dockerfile-dev-with-volumes/pod-front/Dockerfile +++ b/dockerfile-dev-with-volumes/pod-back/Dockerfile @@ -30,12 +30,12 @@ COPY ./requirements-dev.txt . RUN mkdir /tmp/node_modules/ COPY --from=source-build-js /tmp/pod/node_modules/ /tmp/node_modules/ - +# TODO remove ES version - move it into env var RUN pip3 install --no-cache-dir -r requirements-conteneur.txt \ && pip3 install elasticsearch==7.17.7 # ENTRYPOINT : -COPY ./dockerfile-dev-with-volumes/pod/my-entrypoint.sh /tmp/my-entrypoint.sh -RUN chmod 755 /tmp/my-entrypoint.sh +COPY ./dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh /tmp/my-entrypoint-back.sh +RUN chmod 755 /tmp/my-entrypoint-back.sh -ENTRYPOINT ["bash", "/tmp/my-entrypoint.sh"] \ No newline at end of file +ENTRYPOINT ["bash", "/tmp/my-entrypoint-back.sh"] \ No newline at end of file diff --git a/dockerfile-dev-with-volumes/pod-front/my-entrypoint.sh b/dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh similarity index 93% rename from dockerfile-dev-with-volumes/pod-front/my-entrypoint.sh rename to dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh index 4dde88609d..e82f5e59ea 100644 --- a/dockerfile-dev-with-volumes/pod-front/my-entrypoint.sh +++ b/dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh @@ -25,7 +25,7 @@ fi # Le serveur de développement permet de tester vos futures modifications facilement. # N'hésitez pas à lancer le serveur de développement pour vérifier vos modifications au fur et à mesure. # À ce niveau, vous devriez avoir le site en français et en anglais et voir l'ensemble de la page d'accueil. -celery -A pod.video_encode_transcript.importing_tasks worker -l INFO -Q importing --detach -celery -A pod.video_encode_transcript.importing_transcript_tasks worker -l INFO -Q importing_transcript --detach +celery -A pod.video_encode_transcript.importing_tasks worker -l INFO -Q importing --concurrency 1 --detach +celery -A pod.video_encode_transcript.importing_transcript_tasks worker -l INFO -Q importing_transcript --concurrency 1 --detach python3 manage.py runserver 0.0.0.0:8080 --insecure sleep infinity diff --git a/dockerfile-dev-with-volumes/pod-encode/my-entrypoint-encode.sh b/dockerfile-dev-with-volumes/pod-encode/my-entrypoint-encode.sh index 6146fa98df..1ce04e2c59 100644 --- a/dockerfile-dev-with-volumes/pod-encode/my-entrypoint-encode.sh +++ b/dockerfile-dev-with-volumes/pod-encode/my-entrypoint-encode.sh @@ -1,6 +1,6 @@ #!/bin/sh echo "Launching commands into pod-dev" -until nc -z pod-front 8080; do echo waiting for pod-front; sleep 10; done; +until nc -z pod-back 8080; do echo waiting for pod-back; sleep 10; done; # Serveur d'encodage -celery -A pod.video_encode_transcript.encoding_tasks worker -l INFO -Q encoding +celery -A pod.video_encode_transcript.encoding_tasks worker -l INFO -Q encoding --concurrency 1 sleep infinity diff --git a/dockerfile-dev-with-volumes/pod-transcript/my-entrypoint-transcript.sh b/dockerfile-dev-with-volumes/pod-transcript/my-entrypoint-transcript.sh index 1ebc34ddfc..73b1993135 100644 --- a/dockerfile-dev-with-volumes/pod-transcript/my-entrypoint-transcript.sh +++ b/dockerfile-dev-with-volumes/pod-transcript/my-entrypoint-transcript.sh @@ -1,6 +1,6 @@ #!/bin/sh echo "Launching commands into pod-dev" -until nc -z pod-front 8080; do echo waiting for pod-front; sleep 10; done; +until nc -z pod-back 8080; do echo waiting for pod-back; sleep 10; done; # Serveur d'encodage -celery -A pod.video_encode_transcript.transcripting_tasks worker -l INFO -Q transcripting +celery -A pod.video_encode_transcript.transcripting_tasks worker -l INFO -Q transcripting --concurrency 1 sleep infinity diff --git a/pod/video_encode_transcript/importing_transcript_tasks.py b/pod/video_encode_transcript/importing_transcript_tasks.py index c528561f15..18be019999 100644 --- a/pod/video_encode_transcript/importing_transcript_tasks.py +++ b/pod/video_encode_transcript/importing_transcript_tasks.py @@ -1,7 +1,7 @@ from celery import Celery from .. import settings import logging -import io +import os import webvtt logger = logging.getLogger(__name__) @@ -26,13 +26,16 @@ # -A pod.video_encode_transcript.importing_transcript_tasks worker \ # -l INFO -Q importing_transcript @importing_transcript_app.task -def start_importing_transcript_task(video_id, msg, textwebvtt): +def start_importing_transcript_task(video_id, msg, temp_vtt_file): """Start the encoding of the video.""" print("Start the importing transcript of the video ID : %s" % video_id) + print("temp_vtt_file : %s" % temp_vtt_file) from pod.video.models import Video from .transcript import save_vtt_and_notify video_to_encode = Video.objects.get(id=video_id) - textwebvtt = "WEBVTT\\n\\n" + textwebvtt - f = io.StringIO(textwebvtt) - new_vtt = webvtt.read_buffer(f) + filename = os.path.basename(temp_vtt_file) + media_temp_dir = os.path.join(settings.MEDIA_ROOT, "temp") + filepath = os.path.join(media_temp_dir, filename) + new_vtt = webvtt.read(filepath) save_vtt_and_notify(video_to_encode, msg, new_vtt) + os.remove(filepath) diff --git a/pod/video_encode_transcript/transcripting_tasks.py b/pod/video_encode_transcript/transcripting_tasks.py index 2b1124a504..a7cabbc526 100644 --- a/pod/video_encode_transcript/transcripting_tasks.py +++ b/pod/video_encode_transcript/transcripting_tasks.py @@ -2,7 +2,9 @@ # pip3 install webvtt-py # pip3 install redis==4.5.4 from celery import Celery +from tempfile import NamedTemporaryFile import logging +import os # call local settings directly # no need to load pod application to send statement from .. import settings @@ -33,6 +35,16 @@ def start_transcripting_task(video_id, mp3filepath, duration, lang): from .transcript_model import start_transcripting from .importing_transcript_tasks import start_importing_transcript_task print("Start the transcripting of the video %s" % video_id) - msg, webvtt = start_transcripting(mp3filepath, duration, lang) + print(video_id, mp3filepath, duration, lang) + msg, text_webvtt = start_transcripting(mp3filepath, duration, lang) print("End of the transcripting of the video") - start_importing_transcript_task.delay(video_id, msg, str(webvtt)) + media_temp_dir = os.path.join(settings.MEDIA_ROOT, "temp") + if not os.path.exists(media_temp_dir): + os.mkdir(media_temp_dir) + temp_vtt_file = NamedTemporaryFile( + dir=media_temp_dir, + delete=False, + suffix=".vtt" + ) + text_webvtt.save(temp_vtt_file.name) + start_importing_transcript_task.delay(video_id, msg, temp_vtt_file.name)