logging works fine

ivanshin · Jul 7, 2023 · b01a6ab · b01a6ab
1 parent 6af54cf
commit b01a6ab
Show file tree

Hide file tree

Showing 5 changed files with 30 additions and 27 deletions.
diff --git a/components/db_logs_handler.py b/components/db_logs_handler.py
@@ -29,7 +29,6 @@ def __init__(self, database, table, attributes_list):
 
         # Create table if needed
         create_table_sql = 'CREATE TABLE IF NOT EXISTS ' + self.table + ' (' + ((' ' + DEFAULT_DATA_TYPE + ', ').join(self.attributes)) + ' ' + DEFAULT_DATA_TYPE + ');'
-        #print(create_table_sql)
         conn = sqlite3.connect(self.database)
         conn.execute(create_table_sql)
         conn.commit()

diff --git a/components/logs_writer.py b/components/logs_writer.py
@@ -10,33 +10,33 @@
 def configure_loger(APP_CONFIGS) -> logging.Logger:
     logger = logging.getLogger(__name__)
     logger.setLevel(logging.INFO)
-
-    attributes_list = ['asctime', 'levelname', 'message']
+    attributes_list = ['asctime', 'levelname', 'service_name', 'message']
     formatter = logging.Formatter('%(' + ((')s' + db_logs_handler.DEFAULT_SEPARATOR + '%(').join(attributes_list)) + ')s')
 
     if APP_CONFIGS['logs_to_db'] == True:
+        logger.propagate = False
         database =  os.path.join(APP_CONFIGS['logs_db_path'], 'LOGS.db')
-        table = 'log'
+        table = 'asr_logs'
         sql_handler = db_logs_handler.SQLiteHandler(database = database, table = table, attributes_list = attributes_list)
         sql_handler.setLevel(logging.INFO)
         sql_handler.setFormatter(formatter)
         logger.addHandler(sql_handler)
+    else:
+        console_handler = logging.StreamHandler()
+        console_handler.setLevel(logging.DEBUG)
+        console_handler.setFormatter(formatter)
 
-    console_handler = logging.StreamHandler()
-    console_handler.setLevel(logging.DEBUG)
-    console_handler.setFormatter(formatter)
-
-
+
 
-    error_file_handler = logging.FileHandler('error.log')
-    error_file_handler.setLevel(logging.ERROR)
-    error_file_handler.setFormatter(formatter)
+        error_file_handler = logging.FileHandler('error.log')
+        error_file_handler.setLevel(logging.ERROR)
+        error_file_handler.setFormatter(formatter)
 
-    critical_file_handler = logging.FileHandler('critical.log')
-    critical_file_handler.setLevel(logging.CRITICAL)
-    critical_file_handler.setFormatter(formatter)
+        critical_file_handler = logging.FileHandler('critical.log')
+        critical_file_handler.setLevel(logging.CRITICAL)
+        critical_file_handler.setFormatter(formatter)
 
-    logger.addHandler(console_handler)
-    logger.addHandler(error_file_handler)
-    logger.addHandler(critical_file_handler)
+        logger.addHandler(console_handler)
+        logger.addHandler(error_file_handler)
+        logger.addHandler(critical_file_handler)
     return logger
diff --git a/components/noisereducer.py b/components/noisereducer.py
@@ -10,6 +10,8 @@
 import os
 import noisereduce as nr
 
+SERVICE_NAME = 'NOISE_CLEANER'
+
 def reduce_noise(path_to_audio_file: Union[Text,Path], output_dir: Union[Text,Path]) -> None:
     """ Reduce noize from single audio file """
 
@@ -27,8 +29,8 @@ def cleaner_worker(configs_dict, queue, logs_queue) -> None:
     while True:
         if not queue.empty():
             f_path = queue.get()
-            logs_queue.put(f'{f_path} Clean start')
+            logs_queue.put(f'{f_path} Clean start' + '|' + SERVICE_NAME)
             reduce_noise(f_path, configs_dict['clean_audio_dir'])
-            logs_queue.put(f'{f_path} Clean end')
+            logs_queue.put(f'{f_path} Clean end'+'|' + SERVICE_NAME)
             os.remove(f_path)
         pass
diff --git a/components/transcriber.py b/components/transcriber.py
@@ -7,6 +7,8 @@
 import json
 import os
 
+SERVICE_NAME = 'TRANSCRIBER'
+
 def transcribe_audio(
     path_to_audio_file: Union[Text,Path],
     output_dir: Union[Text,Path], 
@@ -25,8 +27,8 @@ def transcriber_worker(configs_dict, queue, logs_queue) -> None:
     while True:
         if not queue.empty():
             f_path = queue.get()
-            logs_queue.put((f'{f_path} Transcribe start'))
+            logs_queue.put(f'{f_path} Transcribe start' + '|' + SERVICE_NAME)
             transcribe_audio(f_path, configs_dict['output_dir'], model)
-            logs_queue.put(f'{f_path} Transcribe end')
+            logs_queue.put(f'{f_path} Transcribe end' + '|' + SERVICE_NAME)
             os.remove(f_path)
         pass
diff --git a/main.py b/main.py
@@ -4,9 +4,8 @@
 from components.transcriber import transcriber_worker
 from components.logs_writer import configure_loger
 import multiprocessing as mp
+import logging 
 
-#TODO: Create correct multiprocessing logging mechanism
-# (https://stackoverflow.com/questions/641420/how-should-i-log-while-using-multiprocessing-in-python)
 APP_CONFIGS = read_configs() # read and validate configuration file
 
 if __name__ == '__main__':
@@ -34,16 +33,17 @@
     transcriber_proc = mp.Process(target= transcriber_worker, args= (APP_CONFIGS, queue_to_transcribe, logs_queue))
     transcriber_proc.daemon= True
     transcriber_proc.start()
-    logger.info('Startup success')
+    logging.LoggerAdapter(logger, {'service_name': 'MAIN'}).info('Startup success')
     try:
         while True:
-            logger.info(logs_queue.get())
+            message= logs_queue.get().split('|')
+            logging.LoggerAdapter(logger, {'service_name': f'{message[1]}'}).info(message[0])
             pass
     except KeyboardInterrupt:
         watchdog_cleaner_proc.terminate()
         cleaner.terminate()
         watchdog_transcribe_proc.terminate()
         transcriber_proc.terminate()
-        logger.info('All processes terminated')
+        logging.LoggerAdapter(logger, {'service_name': 'MAIN'}).info('All processes terminated')