Skip to content

Commit

Permalink
changed to use logger instead of print
Browse files Browse the repository at this point in the history
  • Loading branch information
molotgor committed Jun 25, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature.
1 parent 1c81aa8 commit 2396fe6
Showing 3 changed files with 153 additions and 20 deletions.
3 changes: 2 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
@@ -2,4 +2,5 @@ papermill
aiohttp
aiojobs
aiohttp-swagger
ipykernel
ipykernel
logging
46 changes: 27 additions & 19 deletions server.py
Original file line number Diff line number Diff line change
@@ -27,12 +27,14 @@
import datetime
import asyncio
from argparse import ArgumentParser
import logging

serverStatus: str = 'idle'
notebooksDir: str = '/home/jupyter-notebook/'
resultsDir: str = '/home/jupyter-notebook/results/'
logDir: str = '/home/jupyter-notebook/logs/'
tasks: dict = {}
logger: logging.Logger


def notebooksReg(path):
@@ -56,19 +58,20 @@ def readConf(path: str):
global notebooksDir
global resultsDir
global logDir
global logger
try:
file = open(path, "r")
result = json.load(file)
notebooksDir = result.get('notebooks', notebooksDir)
print('notebooksDir=%s' % notebooksDir)
logger.info('notebooksDir=%s', notebooksDir)
if notebooksDir:
createDir(notebooksDir)
resultsDir = result.get('results', resultsDir)
print('resultsDir=%s' % resultsDir)
logger.info('resultsDir=%s',resultsDir)
if resultsDir:
createDir(resultsDir)
logDir = result.get('logs', logDir)
print('logDir=%s' % logDir)
logger.info('logDir=%s', logDir)
if logDir:
createDir(logDir)
except Exception as e:
@@ -166,8 +169,9 @@ async def reqNotebooks(req: Request):
"200":
description: successful operation. Return string array of available files.
"""
global logger
path = req.rel_url.query.get('path')
print('/files/notebooks?path={path}'.format(path=str(path)))
logger.info('/files/notebooks?path={path}'.format(path=str(path)))
pathConverted = path and replacePathServerToLocal(path)
dirsNote = []
if path:
@@ -202,8 +206,9 @@ async def reqJsons(req: Request):
"200":
description: successful operation. Return string array of available files.
"""
global logger
path = req.rel_url.query.get('path')
print('/files/results?path={path}'.format(path=str(path)))
logger.info('/files/results?path={path}'.format(path=str(path)))
pathConverted = path and replacePathServerToLocal(path)
dirsNote = []
dirsRes = []
@@ -247,8 +252,9 @@ async def reqArguments(req: Request):
"404":
description: requested file doesn't exist.
"""
global logger
path = req.rel_url.query['path']
print('/files?path={path}'.format(path=str(path)))
logger.info('/files?path={path}'.format(path=str(path)))
pathConverted = path and replacePathServerToLocal(path)
if not path or not os.path.isfile(pathConverted):
return web.HTTPNotFound()
@@ -258,28 +264,29 @@ async def reqArguments(req: Request):

async def launchNotebook(input, arguments, file_name, task_id):
global tasks
print('launching notebook {input} with {arguments}'.format(input=input, arguments=arguments))
global logger
logger.info('launching notebook {input} with {arguments}'.format(input=input, arguments=arguments))
logOut: str = (logDir + '/%s.log.ipynb' % file_name) if logDir and file_name else None
try:
with pm.utils.chdir(input[:input.rfind('/')]):
input = input[input.rfind('/')+1:]
pm.execute_notebook(input, logOut, arguments)
print('successfully launched notebook {input}'.format(input=input))
logger.debug('successfully launched notebook {input}'.format(input=input))
if tasks.get(task_id):
tasks[task_id] = {
'status': 'success',
'result': arguments.get('output_path')
}
except Exception as error:
print('failed to launch notebook {input}'.format(input=input))
print(error)
logger.info('failed to launch notebook {input}'.format(input=input))
logger.debug(error)
if tasks.get(task_id):
tasks[task_id] = {
'status': 'failed',
'result': error
}
finally:
print('ended launch notebook {input} with {arguments}'.format(input=input, arguments=arguments))
logger.info('ended launch notebook {input} with {arguments}'.format(input=input, arguments=arguments))


async def reqLaunch(req: Request):
@@ -302,8 +309,9 @@ async def reqLaunch(req: Request):
"""
from uuid import uuid4
global tasks
global logger
path = req.rel_url.query.get('path')
print('/execute?path={path}'.format(path=str(path)))
logger.info('/execute?path={path}'.format(path=str(path)))
if not req.can_read_body:
return web.HTTPBadRequest(reason='body with parameters not present')
path = req.rel_url.query.get('path')
@@ -347,14 +355,13 @@ async def reqResult(req: Request):
description: server is currently busy.
"""
global tasks
global logger
task_id = req.rel_url.query.get('id')
print('/result?id={task_id}'.format(task_id=str(task_id)))
logger.info('/result?id={task_id}'.format(task_id=str(task_id)))
task = tasks.get(task_id, None)
print(task)
if not task:
return web.HTTPNotFound()
status = task.get('status', None)
print(status)
if status == 'in progress':
return web.json_response({'status': status})
elif status == 'success':
@@ -368,7 +375,6 @@ async def reqResult(req: Request):
return web.json_response({'status': status, 'result': content})
elif status == 'failed':
error = task.get('result', Exception())
print(error)
return web.json_response({'status': status, 'result': str(error)})
else:
return web.HTTPNotFound()
@@ -392,8 +398,9 @@ async def reqStop(req: Request):
description: server is currently busy.
"""
global tasks
global logger
task_id = req.rel_url.query.get('id')
print('/stop?id={task_id}'.format(task_id=str(task_id)))
logger.info('/stop?id={task_id}'.format(task_id=str(task_id)))
task = tasks.pop(task_id, None)
try:
if task:
@@ -403,6 +410,8 @@ async def reqStop(req: Request):
return web.HTTPOk()

if __name__ == '__main__':
logging.basicConfig(filename='var/th2/config/log4py.conf', level=logging.DEBUG)
logger=logging.getLogger('th2_common')
parser = ArgumentParser()
parser.add_argument('config')
path = vars(parser.parse_args()).get('config')
@@ -411,7 +420,6 @@ async def reqStop(req: Request):

app = web.Application()


setup(app)
app.router.add_route('GET', "/status", reqStatus)
app.router.add_route('GET', "/files/all", reqFiles)
@@ -422,5 +430,5 @@ async def reqStop(req: Request):
app.router.add_route('GET', "/result", reqResult)
app.router.add_route('POST', "/stop", reqStop)
setup_swagger(app)
print('starting server')
logger.info('starting server')
web.run_app(app)
124 changes: 124 additions & 0 deletions var/th2/config/log4py.conf
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
[loggers]
keys=root,th2_common,aiopika

[handlers]
keys=consoleHandler,fileHandler,metricsHandler

[formatters]
keys=formatter

[logger_root]
level=INFO
handlers=consoleHandler,fileHandler
propagate=0

[logger_th2_common]
level=INFO
qualname=th2_common
handlers=consoleHandler,fileHandler
propagate=0

[logger_aiopika]
level=WARNING
qualname=aio_pika
handlers=consoleHandler,fileHandler
propagate=0

[handler_consoleHandler]
class=StreamHandler
formatter=formatter
args=(sys.stdout,)

[handler_fileHandler]
class=FileHandler
formatter=formatter
args=('../all.log',)

[formatter_formatter]
format=%(asctime)s - %(name)s - %(levelname)s - %(message)sINFO:th2_common:asfg
DEBUG:th2_common:notebooksDir=F:/ExactWork/th2-json-stream-provider-py/notebooks/
DEBUG:th2_common:resultsDir=F:/ExactWork/th2-json-stream-provider-py/results/
DEBUG:th2_common:logDir=F:/ExactWork/th2-json-stream-provider-py/logs/
DEBUG:asyncio:Using proactor: IocpProactor
INFO:th2_common:notebooksDir=F:/ExactWork/th2-json-stream-provider-py/notebooks/
INFO:th2_common:resultsDir=F:/ExactWork/th2-json-stream-provider-py/results/
INFO:th2_common:logDir=F:/ExactWork/th2-json-stream-provider-py/logs/
INFO:th2_common:starting server
DEBUG:asyncio:Using proactor: IocpProactor
INFO:th2_common:notebooksDir=F:/ExactWork/th2-json-stream-provider-py/notebooks/
INFO:th2_common:resultsDir=F:/ExactWork/th2-json-stream-provider-py/results/
INFO:th2_common:logDir=F:/ExactWork/th2-json-stream-provider-py/logs/
INFO:th2_common:starting server
DEBUG:asyncio:Using proactor: IocpProactor
INFO:th2_common:/files?path=F:/ExactWork/th2-json-stream-provider-py/notebooks/test_files.ipynb
INFO:papermill:Input Notebook: F:/ExactWork/th2-json-stream-provider-py/notebooks/test_files.ipynb
INFO:aiohttp.access:127.0.0.1 [25/Jun/2024:15:03:30 +0400] "GET /files?path=F:/ExactWork/th2-json-stream-provider-py/notebooks/test_files.ipynb HTTP/1.1" 200 377 "http://localhost:9001/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
INFO:th2_common:/execute?path=F:/ExactWork/th2-json-stream-provider-py/notebooks/test_files.ipynb
INFO:aiohttp.access:127.0.0.1 [25/Jun/2024:15:03:46 +0400] "POST /execute?path=F:/ExactWork/th2-json-stream-provider-py/notebooks/test_files.ipynb HTTP/1.1" 200 295 "http://localhost:9001/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
INFO:th2_common:launching notebook F:/ExactWork/th2-json-stream-provider-py/notebooks/test_files.ipynb with {'test': '20240531-21:50:15.08000000', 'output_path': 'F:/ExactWork/th2-json-stream-provider-py/results//test_files_2024-06-25T11-03-46-991535.jsonl'}
INFO:papermill:Input Notebook: test_files.ipynb
INFO:papermill:Output Notebook: F:/ExactWork/th2-json-stream-provider-py/logs//test_files_2024-06-25T11-03-46-991535.log.ipynb
DEBUG:papermill.translators:Black is not installed, parameters won't be formatted
DEBUG:asyncio:Using selector: SelectSelector
DEBUG:traitlets:Instantiating kernel 'Python 3 (ipykernel)' with kernel provisioner: local-provisioner
DEBUG:traitlets:Starting kernel: ['C:\\Program Files\\Python310\\python.exe', '-m', 'ipykernel_launcher', '-f', 'C:\\Users\\molotgor\\AppData\\Local\\Temp\\tmppk0_9qto.json', '--HistoryManager.hist_file=:memory:']
DEBUG:traitlets:Connecting to: tcp://127.0.0.1:63424
DEBUG:traitlets:connecting iopub channel to tcp://127.0.0.1:63421
DEBUG:traitlets:Connecting to: tcp://127.0.0.1:63421
DEBUG:traitlets:connecting shell channel to tcp://127.0.0.1:63420
DEBUG:traitlets:Connecting to: tcp://127.0.0.1:63420
DEBUG:traitlets:connecting stdin channel to tcp://127.0.0.1:63422
DEBUG:traitlets:Connecting to: tcp://127.0.0.1:63422
DEBUG:traitlets:connecting heartbeat channel to tcp://127.0.0.1:63423
DEBUG:asyncio:Using selector: SelectSelector
DEBUG:traitlets:connecting control channel to tcp://127.0.0.1:63424
DEBUG:traitlets:Connecting to: tcp://127.0.0.1:63424
INFO:papermill:Executing notebook with kernel: python3
DEBUG:papermill:Executing cell:
output_path='test_file.jsonl'
test = ''
DEBUG:papermill:msg_type: status
DEBUG:papermill:content: {'execution_state': 'busy'}
DEBUG:papermill:msg_type: execute_input
DEBUG:papermill:content: {'code': "output_path='test_file.jsonl'\ntest = ''", 'execution_count': 1}
DEBUG:papermill:msg_type: status
DEBUG:papermill:content: {'execution_state': 'idle'}
DEBUG:papermill:Executing cell:
# Parameters
test = "20240531-21:50:15.08000000"
output_path = "F:/ExactWork/th2-json-stream-provider-py/results//test_files_2024-06-25T11-03-46-991535.jsonl"

DEBUG:papermill:msg_type: status
DEBUG:papermill:content: {'execution_state': 'busy'}
DEBUG:papermill:msg_type: execute_input
DEBUG:papermill:content: {'code': '# Parameters\ntest = "20240531-21:50:15.08000000"\noutput_path = "F:/ExactWork/th2-json-stream-provider-py/results//test_files_2024-06-25T11-03-46-991535.jsonl"\n', 'execution_count': 2}
DEBUG:papermill:msg_type: status
DEBUG:papermill:content: {'execution_state': 'idle'}
DEBUG:papermill:Executing cell:
input_path='filetoinput.jsonl'
DEBUG:papermill:msg_type: status
DEBUG:papermill:content: {'execution_state': 'busy'}
DEBUG:papermill:msg_type: execute_input
DEBUG:papermill:content: {'code': "input_path='filetoinput.jsonl'", 'execution_count': 3}
DEBUG:papermill:msg_type: status
DEBUG:papermill:content: {'execution_state': 'idle'}
DEBUG:papermill:Executing cell:
import json
import asyncio

with open(output_path, "w") as out_file:
with open(input_path, "r") as in_file:
for i in range(6000):
json.dump({'a':i}, out_file)
out_file.write("\n")
DEBUG:papermill:msg_type: status
DEBUG:papermill:content: {'execution_state': 'busy'}
DEBUG:papermill:msg_type: execute_input
DEBUG:papermill:content: {'code': 'import json\nimport asyncio\n\nwith open(output_path, "w") as out_file:\n with open(input_path, "r") as in_file:\n for i in range(6000):\n json.dump({\'a\':i}, out_file)\n out_file.write("\\n")', 'execution_count': 4}
DEBUG:papermill:msg_type: status
DEBUG:papermill:content: {'execution_state': 'idle'}
DEBUG:traitlets:Destroying zmq context for <jupyter_client.asynchronous.client.AsyncKernelClient object at 0x000001A043C1AB90>
DEBUG:th2_common:successfully launched notebook test_files.ipynb
INFO:th2_common:ended launch notebook test_files.ipynb with {'test': '20240531-21:50:15.08000000', 'output_path': 'F:/ExactWork/th2-json-stream-provider-py/results//test_files_2024-06-25T11-03-46-991535.jsonl'}
INFO:th2_common:/result?id=9d716019-40f0-4faa-8216-aa6ecdffa277
INFO:aiohttp.access:127.0.0.1 [25/Jun/2024:15:03:52 +0400] "GET /result?id=9d716019-40f0-4faa-8216-aa6ecdffa277 HTTP/1.1" 200 89105 "http://localhost:9001/" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"

0 comments on commit 2396fe6

Please sign in to comment.