logging queue complete

ivanshin · Jul 5, 2023 · 6af54cf · 6af54cf
1 parent 1944d16
commit 6af54cf
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 10 deletions.
diff --git a/components/noisereducer.py b/components/noisereducer.py
@@ -22,13 +22,13 @@ def reduce_noise(path_to_audio_file: Union[Text,Path], output_dir: Union[Text,Pa
     wavfile.write(os.path.join(output_dir, ts + "_" + file_name), rate, reduced_noise)
     return None
 
-def cleaner_worker(configs_dict, queue, logger) -> None:
+def cleaner_worker(configs_dict, queue, logs_queue) -> None:
     """ Daemon cleaner worker """
     while True:
         if not queue.empty():
             f_path = queue.get()
-            logger.info(f'{f_path} Clean start')
+            logs_queue.put(f'{f_path} Clean start')
             reduce_noise(f_path, configs_dict['clean_audio_dir'])
-            logger.info(f'{f_path} Clean end')
+            logs_queue.put(f'{f_path} Clean end')
             os.remove(f_path)
         pass
diff --git a/components/transcriber.py b/components/transcriber.py
@@ -19,14 +19,14 @@ def transcribe_audio(
         json.dump(transcription, out_file, ensure_ascii= False)
     return None
 
-def transcriber_worker(configs_dict, queue, logger) -> None:
+def transcriber_worker(configs_dict, queue, logs_queue) -> None:
     """ Daemon cleaner worker """
     model = SpeechRecognitionModel("jonatasgrosman/wav2vec2-large-xlsr-53-russian")
     while True:
         if not queue.empty():
             f_path = queue.get()
-            logger.info(f'{f_path} Transcribe start')
+            logs_queue.put((f'{f_path} Transcribe start'))
             transcribe_audio(f_path, configs_dict['output_dir'], model)
-            logger.info(f'{f_path} Transcribe end')
+            logs_queue.put(f'{f_path} Transcribe end')
             os.remove(f_path)
         pass
diff --git a/main.py b/main.py
@@ -12,8 +12,9 @@
 if __name__ == '__main__':
     logger = configure_loger(APP_CONFIGS)
     # variables
-    queue_to_cleaning = mp.Queue()
-    queue_to_transcribe = mp.Queue()
+    logs_queue = mp.Queue(-1)
+    queue_to_cleaning = mp.Queue(-1)
+    queue_to_transcribe = mp.Queue(-1)
 
     # subprocesses
     # 1) watchdog with queue to cleaning
@@ -22,20 +23,21 @@
     watchdog_cleaner_proc.start()
     #watchdog_proc.join()
     # 2) cleaner
-    cleaner = mp.Process(target= cleaner_worker, args= (APP_CONFIGS, queue_to_cleaning, logger))
+    cleaner = mp.Process(target= cleaner_worker, args= (APP_CONFIGS, queue_to_cleaning, logs_queue))
     cleaner.daemon= True
     cleaner.start()
     # 3) watchdog with queue to transcribation
     watchdog_transcribe_proc = mp.Process(target= create_observer, args= (APP_CONFIGS['clean_audio_dir'], queue_to_transcribe))
     watchdog_transcribe_proc.daemon= True
     watchdog_transcribe_proc.start()
     # 4) Russian wav2vec implementation
-    transcriber_proc = mp.Process(target= transcriber_worker, args= (APP_CONFIGS, queue_to_transcribe, logger))
+    transcriber_proc = mp.Process(target= transcriber_worker, args= (APP_CONFIGS, queue_to_transcribe, logs_queue))
     transcriber_proc.daemon= True
     transcriber_proc.start()
     logger.info('Startup success')
     try:
         while True:
+            logger.info(logs_queue.get())
             pass
     except KeyboardInterrupt:
         watchdog_cleaner_proc.terminate()