Fix double logging issue

database_functions.py

@@ -1,5 +1,5 @@
 from embeddings_data_models import Base, TextEmbedding, DocumentEmbedding, Document, TokenLevelEmbedding, TokenLevelEmbeddingBundle, TokenLevelEmbeddingBundleCombinedFeatureVector, AudioTranscript
-from logger_config import setup_logger
+from logger_config import logger
 import traceback
 import asyncio
 import random
@@ -11,7 +11,6 @@
 from datetime import datetime
 from decouple import config
 
-logger = setup_logger()
 db_writer = None
 DATABASE_URL = "sqlite+aiosqlite:///swiss_army_llama.sqlite"
 MAX_RETRIES = config("MAX_RETRIES", default=3, cast=int)

logger_config.py

@@ -13,7 +13,7 @@ def setup_logger():
     if not os.path.exists(old_logs_dir):
         os.makedirs(old_logs_dir)
     logger.setLevel(logging.INFO)
-    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
     log_file_path = 'swiss_army_llama.log'
     log_queue = queue.Queue(-1)  # Create a queue for the handlers
     fh = RotatingFileHandler(log_file_path, maxBytes=10*1024*1024, backupCount=5)
@@ -24,12 +24,10 @@ def rotator(source, dest):
         shutil.move(source, dest)
     fh.namer = namer
     fh.rotator = rotator
-    sh = logging.StreamHandler()  # Stream handler
-    sh.setFormatter(formatter)
     queue_handler = QueueHandler(log_queue)  # Create QueueHandler
     queue_handler.setFormatter(formatter)
     logger.addHandler(queue_handler)
-    listener = QueueListener(log_queue, fh, sh)  # Create QueueListener with real handlers
+    listener = QueueListener(log_queue, fh)  # Create QueueListener with only the file handler
     listener.start()
     logging.getLogger('sqlalchemy.engine').setLevel(logging.WARNING)  # Configure SQLalchemy logging
-    return logger
+    return logger

misc_utility_functions.py

@@ -1,4 +1,4 @@
-from logger_config import setup_logger
+from logger_config import logger
 from database_functions import AsyncSessionLocal
 import socket
 import os
@@ -12,8 +12,6 @@
 from collections import defaultdict
 from sqlalchemy import text as sql_text
 
-logger = setup_logger()
-
 class suppress_stdout_stderr(object):
     def __enter__(self):
         self.outnull_file = open(os.devnull, 'w')

ramdisk_functions.py

@@ -1,11 +1,10 @@
-from logger_config import setup_logger
+from logger_config import logger
 import os
 import subprocess
 import shutil
 import psutil
 from decouple import config
 
-logger = setup_logger()
 RAMDISK_PATH = config("RAMDISK_PATH", default="/mnt/ramdisk", cast=str)
 RAMDISK_SIZE_IN_GB = config("RAMDISK_SIZE_IN_GB", default=1, cast=int)
 

service_functions.py

@@ -1,4 +1,4 @@
-from logger_config import setup_logger
+from logger_config import logger
 import shared_resources
 from shared_resources import load_model, token_level_embedding_model_cache, text_completion_model_cache, is_gpu_available
 from database_functions import AsyncSessionLocal, DatabaseWriter, execute_with_retry
@@ -31,7 +31,7 @@
 from decouple import config
 from faster_whisper import WhisperModel
 from llama_cpp import Llama, LlamaGrammar
-logger = setup_logger()
+
 SWISS_ARMY_LLAMA_SERVER_LISTEN_PORT = config("SWISS_ARMY_LLAMA_SERVER_LISTEN_PORT", default=8089, cast=int)
 DEFAULT_MODEL_NAME = config("DEFAULT_MODEL_NAME", default="openchat_v3.2_super", cast=str) 
 LLM_CONTEXT_SIZE_IN_TOKENS = config("LLM_CONTEXT_SIZE_IN_TOKENS", default=512, cast=int)

shared_resources.py

@@ -1,7 +1,7 @@
 from misc_utility_functions import  is_redis_running, build_faiss_indexes, suppress_stdout_stderr
 from database_functions import DatabaseWriter, initialize_db
 from ramdisk_functions import setup_ramdisk, copy_models_to_ramdisk, check_that_user_has_required_permissions_to_manage_ramdisks
-from logger_config import setup_logger
+from logger_config import logger
 from aioredlock import Aioredlock
 import aioredis
 import asyncio
@@ -17,7 +17,6 @@
 from decouple import config
 from fastapi import HTTPException
 
-logger = setup_logger()
 embedding_model_cache = {} # Model cache to store loaded models
 token_level_embedding_model_cache = {} # Model cache to store loaded token-level embedding models
 text_completion_model_cache = {} # Model cache to store loaded text completion models