diff --git a/local-run/with-jupyter-notebook/compose.yml b/local-run/with-jupyter-notebook/compose.yml index acb7cd5..fd52ab1 100644 --- a/local-run/with-jupyter-notebook/compose.yml +++ b/local-run/with-jupyter-notebook/compose.yml @@ -8,7 +8,7 @@ services: - jupyter_volume:/home/jovyan/:rw - python_lib:/home/json-stream/python/lib:ro - ${USER_DATA_DIR}:/home/jovyan/user-data/:rw - - ./json-stream-provider/custom.json:/var/th2/config/custom.json:ro + - ./json-stream-provider:/var/th2/config:ro networks: - th2_network diff --git a/local-run/with-jupyter-notebook/json-stream-provider/log4py.conf b/local-run/with-jupyter-notebook/json-stream-provider/log4py.conf new file mode 100644 index 0000000..0631925 --- /dev/null +++ b/local-run/with-jupyter-notebook/json-stream-provider/log4py.conf @@ -0,0 +1,22 @@ +[loggers] +keys=root + +[handlers] +keys=consoleHandler + +[formatters] +keys=formatter + +[logger_root] +level=INFO +handlers=consoleHandler + +[handler_consoleHandler] +class=StreamHandler +level=INFO +formatter=formatter +args=(sys.stdout,) + +[formatter_formatter] +format=%(asctime)s - %(name)s - %(levelname)s - %(message)s +datefmt=%Y-%m-%d %H:%M:%S \ No newline at end of file diff --git a/log_configuratior.py b/log_configuratior.py new file mode 100644 index 0000000..cec3dc9 --- /dev/null +++ b/log_configuratior.py @@ -0,0 +1,34 @@ +import logging.config +import os + +log4py_file = '/var/th2/config/log4py.conf' +def configureLogging(): + if os.path.exists(log4py_file): + logging.config.fileConfig(log4py_file, disable_existing_loggers=False) + logging.getLogger(__name__).info(f'Logger is configured by {log4py_file} file') + else: + default_logging_config = { + 'version': 1, + 'formatters': { + 'default': { + 'format': '%(asctime)s - %(name)s - %(levelname)s - %(message)s', + 'datefmt': '%Y-%m-%d %H:%M:%S' + }, + }, + 'handlers': { + 'console': { + 'class': 'logging.StreamHandler', + 'formatter': 'default', + 'level': 'DEBUG', + 'stream': 'ext://sys.stdout' + }, + }, + 'root': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, + } + logging.config.dictConfig(default_logging_config) + logging.getLogger(__name__).info('Logger is configured by default') + + diff --git a/server.py b/server.py index 989639a..78882f8 100644 --- a/server.py +++ b/server.py @@ -13,29 +13,30 @@ # limitations under the License. import os + +from log_configuratior import configureLogging + os.system('pip list') -import subprocess -import sys import papermill as pm from aiohttp.web_request import Request from aiohttp import web from aiojobs.aiohttp import setup, spawn from aiohttp_swagger import * -from glob import glob import json import datetime import asyncio from argparse import ArgumentParser -import logging +import logging.config serverStatus: str = 'idle' notebooksDir: str = '/home/jupyter-notebook/' resultsDir: str = '/home/jupyter-notebook/results/' logDir: str = '/home/jupyter-notebook/logs/' tasks: dict = {} -logger: logging.Logger +configureLogging() +logger: logging.Logger = logging.getLogger('j-sp') def notebooksReg(path): return path + '/*.ipynb' @@ -75,7 +76,7 @@ def readConf(path: str): if logDir: createDir(logDir) except Exception as e: - logger.debug(e) + logger.error(f"Read '{path}' configuration failure", e) async def reqStatus(req: Request): @@ -263,15 +264,14 @@ async def launchNotebook(input, arguments, file_name, task_id): with pm.utils.chdir(input[:input.rfind('/')]): input = input[input.rfind('/')+1:] pm.execute_notebook(input, logOut, arguments) - logger.debug('successfully launched notebook {input}'.format(input=input)) + logger.info('successfully launched notebook {input}'.format(input=input)) if tasks.get(task_id): tasks[task_id] = { 'status': 'success', 'result': arguments.get('output_path') } except Exception as error: - logger.info('failed to launch notebook {input}'.format(input=input)) - logger.debug(error) + logger.error('failed to launch notebook {input}'.format(input=input), error) if tasks.get(task_id): tasks[task_id] = { 'status': 'failed', @@ -430,8 +430,6 @@ async def reqStop(req: Request): return web.HTTPOk() if __name__ == '__main__': - logging.basicConfig(filename='var/th2/config/log4py.conf', level=logging.DEBUG) - logger=logging.getLogger('th2_common') parser = ArgumentParser() parser.add_argument('config') path = vars(parser.parse_args()).get('config') diff --git a/var/th2/config/log4py.conf b/var/th2/config/log4py.conf deleted file mode 100644 index 1b2a653..0000000 --- a/var/th2/config/log4py.conf +++ /dev/null @@ -1,38 +0,0 @@ -[loggers] -keys=root,th2_common,aiopika - -[handlers] -keys=consoleHandler,fileHandler,metricsHandler - -[formatters] -keys=formatter - -[logger_root] -level=INFO -handlers=consoleHandler,fileHandler -propagate=0 - -[logger_th2_common] -level=INFO -qualname=th2_common -handlers=consoleHandler,fileHandler -propagate=0 - -[logger_aiopika] -level=WARNING -qualname=aio_pika -handlers=consoleHandler,fileHandler -propagate=0 - -[handler_consoleHandler] -class=StreamHandler -formatter=formatter -args=(sys.stdout,) - -[handler_fileHandler] -class=FileHandler -formatter=formatter -args=('../all.log',) - -[formatter_formatter] -format=%(asctime)s - %(name)s - %(levelname)s - %(message)s \ No newline at end of file