use file to temporarily store webvtt - rename pod-front docker to pod…

…-back (front is nginx)

docker-compose-full-dev-with-volumes.yml

@@ -7,11 +7,11 @@ x-elasticsearch-volumes: &elasticsearch-volumes
 version: '3.7'
 
 services:
- pod-front:
- container_name: pod-front-with-volumes
+ pod-back:
+ container_name: pod-back-with-volumes
  build:
  context: .
- dockerfile: dockerfile-dev-with-volumes/pod-front/Dockerfile
+ dockerfile: dockerfile-dev-with-volumes/pod-back/Dockerfile
  depends_on:
  - elasticsearch
  - redis
@@ -27,7 +27,7 @@ services:
  context: .
  dockerfile: dockerfile-dev-with-volumes/pod-encode/Dockerfile
  depends_on:
- - pod-front
+ - pod-back
  env_file:
  - ./.env.dev
  volumes: *pod-volumes
@@ -38,7 +38,7 @@ services:
  context: .
  dockerfile: dockerfile-dev-with-volumes/pod-transcript/Dockerfile
  depends_on:
- - pod-front
+ - pod-back
  env_file:
  - ./.env.dev
  volumes: *pod-volumes

dockerfile-dev-with-volumes/pod-front/Dockerfile → dockerfile-dev-with-volumes/pod-back/Dockerfile

@@ -30,12 +30,12 @@ COPY ./requirements-dev.txt .
 RUN mkdir /tmp/node_modules/
 
 COPY --from=source-build-js /tmp/pod/node_modules/ /tmp/node_modules/
-
+# TODO remove ES version - move it into env var
 RUN pip3 install --no-cache-dir -r requirements-conteneur.txt \
  && pip3 install elasticsearch==7.17.7
 
 # ENTRYPOINT :
-COPY ./dockerfile-dev-with-volumes/pod/my-entrypoint.sh /tmp/my-entrypoint.sh
-RUN chmod 755 /tmp/my-entrypoint.sh
+COPY ./dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh /tmp/my-entrypoint-back.sh
+RUN chmod 755 /tmp/my-entrypoint-back.sh
 
-ENTRYPOINT ["bash", "/tmp/my-entrypoint.sh"]
+ENTRYPOINT ["bash", "/tmp/my-entrypoint-back.sh"]

dockerfile-dev-with-volumes/pod-front/my-entrypoint.sh → dockerfile-dev-with-volumes/pod-back/my-entrypoint-back.sh

@@ -25,7 +25,7 @@ fi
 # Le serveur de développement permet de tester vos futures modifications facilement.
 # N'hésitez pas à lancer le serveur de développement pour vérifier vos modifications au fur et à mesure.
 # À ce niveau, vous devriez avoir le site en français et en anglais et voir l'ensemble de la page d'accueil.
-celery -A pod.video_encode_transcript.importing_tasks worker -l INFO -Q importing --detach
-celery -A pod.video_encode_transcript.importing_transcript_tasks worker -l INFO -Q importing_transcript --detach
+celery -A pod.video_encode_transcript.importing_tasks worker -l INFO -Q importing --concurrency 1 --detach
+celery -A pod.video_encode_transcript.importing_transcript_tasks worker -l INFO -Q importing_transcript --concurrency 1 --detach
 python3 manage.py runserver 0.0.0.0:8080 --insecure
 sleep infinity

dockerfile-dev-with-volumes/pod-encode/my-entrypoint-encode.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 echo "Launching commands into pod-dev"
-until nc -z pod-front 8080; do echo waiting for pod-front; sleep 10; done;
+until nc -z pod-back 8080; do echo waiting for pod-back; sleep 10; done;
 # Serveur d'encodage
-celery -A pod.video_encode_transcript.encoding_tasks worker -l INFO -Q encoding
+celery -A pod.video_encode_transcript.encoding_tasks worker -l INFO -Q encoding --concurrency 1 
 sleep infinity

dockerfile-dev-with-volumes/pod-transcript/my-entrypoint-transcript.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 echo "Launching commands into pod-dev"
-until nc -z pod-front 8080; do echo waiting for pod-front; sleep 10; done;
+until nc -z pod-back 8080; do echo waiting for pod-back; sleep 10; done;
 # Serveur d'encodage
-celery -A pod.video_encode_transcript.transcripting_tasks worker -l INFO -Q transcripting
+celery -A pod.video_encode_transcript.transcripting_tasks worker -l INFO -Q transcripting --concurrency 1 
 sleep infinity

pod/video_encode_transcript/importing_transcript_tasks.py

@@ -1,7 +1,7 @@
 from celery import Celery
 from .. import settings
 import logging
-import io
+import os
 import webvtt
 logger = logging.getLogger(__name__)
 
@@ -26,13 +26,16 @@
 # -A pod.video_encode_transcript.importing_transcript_tasks worker \
 # -l INFO -Q importing_transcript
 @importing_transcript_app.task
-def start_importing_transcript_task(video_id, msg, textwebvtt):
+def start_importing_transcript_task(video_id, msg, temp_vtt_file):
  """Start the encoding of the video."""
  print("Start the importing transcript of the video ID : %s" % video_id)
+ print("temp_vtt_file : %s" % temp_vtt_file)
  from pod.video.models import Video
  from .transcript import save_vtt_and_notify
  video_to_encode = Video.objects.get(id=video_id)
- textwebvtt = "WEBVTT\\n\\n" + textwebvtt
- f = io.StringIO(textwebvtt)
- new_vtt = webvtt.read_buffer(f)
+ filename = os.path.basename(temp_vtt_file)
+ media_temp_dir = os.path.join(settings.MEDIA_ROOT, "temp")
+ filepath = os.path.join(media_temp_dir, filename)
+ new_vtt = webvtt.read(filepath)
  save_vtt_and_notify(video_to_encode, msg, new_vtt)
+ os.remove(filepath)

pod/video_encode_transcript/transcripting_tasks.py

@@ -2,7 +2,9 @@
 # pip3 install webvtt-py
 # pip3 install redis==4.5.4
 from celery import Celery
+from tempfile import NamedTemporaryFile
 import logging
+import os
 # call local settings directly
 # no need to load pod application to send statement
 from .. import settings
@@ -33,6 +35,16 @@ def start_transcripting_task(video_id, mp3filepath, duration, lang):
  from .transcript_model import start_transcripting
  from .importing_transcript_tasks import start_importing_transcript_task
  print("Start the transcripting of the video %s" % video_id)
- msg, webvtt = start_transcripting(mp3filepath, duration, lang)
+ print(video_id, mp3filepath, duration, lang)
+ msg, text_webvtt = start_transcripting(mp3filepath, duration, lang)
  print("End of the transcripting of the video")
- start_importing_transcript_task.delay(video_id, msg, str(webvtt))
+ media_temp_dir = os.path.join(settings.MEDIA_ROOT, "temp")
+ if not os.path.exists(media_temp_dir):
+ os.mkdir(media_temp_dir)
+ temp_vtt_file = NamedTemporaryFile(
+ dir=media_temp_dir,
+ delete=False,
+ suffix=".vtt"
+ )
+ text_webvtt.save(temp_vtt_file.name)
+ start_importing_transcript_task.delay(video_id, msg, temp_vtt_file.name)