-
Notifications
You must be signed in to change notification settings - Fork 0
/
ml_worker.py
42 lines (31 loc) · 1.32 KB
/
ml_worker.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
import os, sys
import redis
from rq import Worker, Queue, Connection
import logging, logging.config, yaml
from logging.handlers import RotatingFileHandler
import core.db as dbHandler
logger = None
listen = ['Q3']
redis_url = os.getenv('REDISTOGO_URL', 'redis://localhost:6379')
conn3 = redis.from_url(redis_url)
def ml_exception_handler(job, exc_type, exc_value, traceback):
account = job.id.split('-')[0].strip()
logger.error('[{}] job {} execution failed. status: {}'.format(account, job.id, job.get_status()))
dbHandler.update_queue_status(account, 3, dbHandler.FAILED)
return False
if __name__ == '__main__':
# TODO: move log file name to config file
logger = logging.getLogger('rq.worker')
# configure file handler
rfh = RotatingFileHandler('logs/worker_ml.log', maxBytes=10*1024*1024, backupCount=10)
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
rfh.setFormatter(formatter)
logger.addHandler(rfh)
# configure stream handler
sh = logging.StreamHandler()
sh.setFormatter(formatter)
logging.getLogger().addHandler(sh)
#logging.config.dictConfig(yaml.load(open('./config/logging-workers.conf')))
with Connection(conn3):
worker = Worker(list(map(Queue, listen)), exception_handlers=[ml_exception_handler])
worker.work()