From 5e97622825e0bc3ead0e989bab290840e11ab9be Mon Sep 17 00:00:00 2001 From: Mayowa Makinde Date: Wed, 8 Nov 2017 10:22:43 +0100 Subject: [PATCH 1/2] wrap external calls with try-except --- healthtools/core.py | 11 ++++++++ healthtools/search/elastic.py | 24 +++++++++++------ healthtools/search/nurses.py | 41 +++++++++++++++++------------- healthtools/search/query.py | 6 +++++ healthtools/views/search_api.py | 22 +++++++++++----- healthtools/manage.py => manage.py | 0 6 files changed, 73 insertions(+), 31 deletions(-) rename healthtools/manage.py => manage.py (100%) diff --git a/healthtools/core.py b/healthtools/core.py index 652f930..82eb0ec 100644 --- a/healthtools/core.py +++ b/healthtools/core.py @@ -63,3 +63,14 @@ def __call__(self, req): req = super(AWS4AuthNotUnicode, self).__call__(req) req.headers = {str(name): value for name, value in req.headers.items()} return req + +def print_error(message): + ''' + Print error messages in the terminal. + ''' + error = "- ERROR: " + message['ERROR'] + source = ("- SOURCE: " + message['SOURCE']) if "SOURCE" in message else "" + error_msg = "- MESSAGE: " + message['MESSAGE'] + msg = "\n".join([error, source, error_msg]) + + log.error(msg) diff --git a/healthtools/search/elastic.py b/healthtools/search/elastic.py index 53351f2..556da0a 100644 --- a/healthtools/search/elastic.py +++ b/healthtools/search/elastic.py @@ -1,14 +1,22 @@ -from healthtools.core import es, es_index +from healthtools.core import es, es_index, print_error def search(query, doc_type): - result = es.search( - index=es_index, - body={'query': match_all_text(query)}, - doc_type=doc_type - ) - hits = result.get('hits', {}) - return hits + try: + result = es.search( + index=es_index, + body={'query': match_all_text(query)}, + doc_type=doc_type + ) + hits = result.get('hits', {}) + return hits + except Exception as err: + error = { + "ERROR": "Elastic Search", + "MESSAGE": str(err) + } + print_error(error) + def match_all(): diff --git a/healthtools/search/nurses.py b/healthtools/search/nurses.py index 9cc02f9..41886d6 100644 --- a/healthtools/search/nurses.py +++ b/healthtools/search/nurses.py @@ -1,5 +1,6 @@ import requests from bs4 import BeautifulSoup +from healthtools.core import print_error NURSING_COUNCIL_URL = 'http://nckenya.com/services/search.php?p=1&s={}' @@ -16,26 +17,32 @@ def get_nurses_from_nc_registry(query): Get nurses from the nursing council of Kenya registry ''' url = NURSING_COUNCIL_URL.format(query) - response = requests.get(url) nurses = {'hits': [], 'total': 0} + try: + response = requests.get(url) + if 'No results' in response.content: + return nurses - if 'No results' in response.content: - return nurses - - # make soup for parsing out of response and get the table - soup = BeautifulSoup(response.content, 'html.parser') - table = soup.find('table', {'class': 'zebra'}).find('tbody') - rows = table.find_all("tr") + # make soup for parsing out of response and get the table + soup = BeautifulSoup(response.content, 'html.parser') + table = soup.find('table', {'class': 'zebra'}).find('tbody') + rows = table.find_all("tr") - # parse table for the nurses data - for row in rows: - # only the columns we want - columns = row.find_all('td')[:len(NURSES_FIELDS)] - columns = [text.text.strip() for text in columns] + # parse table for the nurses data + for row in rows: + # only the columns we want + columns = row.find_all('td')[:len(NURSES_FIELDS)] + columns = [text.text.strip() for text in columns] - entry = dict(zip(NURSES_FIELDS, columns)) - nurses['hits'].append(entry) + entry = dict(zip(NURSES_FIELDS, columns)) + nurses['hits'].append(entry) - nurses['total'] = len(nurses['hits']) + nurses['total'] = len(nurses['hits']) - return nurses + return nurses + except Exception as err: + error = { + "ERROR": "get_nurses_from_nc_registry()", + "MESSAGE": str(err) + } + print_error(error) diff --git a/healthtools/search/query.py b/healthtools/search/query.py index 46c2848..b7b2301 100644 --- a/healthtools/search/query.py +++ b/healthtools/search/query.py @@ -3,6 +3,7 @@ from healthtools.settings import WIT_ACCESS_TOKEN from healthtools.documents import DOCUMENTS, doc_exists +from healthtools.core import print_error from healthtools.search import elastic, nurses @@ -55,6 +56,11 @@ def determine_doc_type(query, doc_type=None): for keyword in DOCUMENTS[doc]['keywords']: if query.startswith(keyword + ' '): return doc, DOCUMENTS[doc]['search_type'] + error = { + "ERROR": "doc_type could not be determined from query", + "MESSAGE": 'Query supplied = ' + query + } + print_error(error) return False, False diff --git a/healthtools/views/search_api.py b/healthtools/views/search_api.py index 8fe6623..82d2930 100644 --- a/healthtools/views/search_api.py +++ b/healthtools/views/search_api.py @@ -1,6 +1,7 @@ from flask import Blueprint, request, jsonify from healthtools.search import run_query +from healthtools.core import print_error blueprint = Blueprint('search_api', __name__) @@ -9,17 +10,26 @@ def index(doc_type=None): query = request.args.get('q') - result, doc_type = run_query(query, doc_type) + try: + result, doc_type = run_query(query, doc_type) + response = jsonify({ + 'result': result, + 'doc_type': doc_type, + 'status': 'OK' + }) - # Error with run_query (run_query returns false) - if not result: - return jsonify({ + except Exception as err: + response = jsonify({ 'result': {'hits': [], 'total': 0}, 'doc_type': doc_type, 'status': 'FAILED', 'msg': '' # TODO: Pass run_query message here. }) + error = { + "ERROR": "index()", + "MESSAGE": str(err) + } + print_error(error) # TODO: Log event here (send to Google Analytics) - - return jsonify({'result': result, 'doc_type': doc_type, 'status': 'OK'}) + return response diff --git a/healthtools/manage.py b/manage.py similarity index 100% rename from healthtools/manage.py rename to manage.py From 98dd661e665a72816f3dfabd271bc4b214d0090e Mon Sep 17 00:00:00 2001 From: Mayowa Makinde Date: Mon, 13 Nov 2017 17:48:07 +0100 Subject: [PATCH 2/2] remove print_error function --- healthtools/__init__.py | 4 +++- healthtools/core.py | 11 ----------- manage.py => healthtools/manage.py | 0 healthtools/search/elastic.py | 12 +++++------- healthtools/search/nurses.py | 9 +++------ healthtools/search/query.py | 11 +++-------- healthtools/views/search_api.py | 9 +++------ 7 files changed, 17 insertions(+), 39 deletions(-) rename manage.py => healthtools/manage.py (100%) diff --git a/healthtools/__init__.py b/healthtools/__init__.py index 1781b68..a30c974 100644 --- a/healthtools/__init__.py +++ b/healthtools/__init__.py @@ -1,4 +1,6 @@ import logging # loggers. -logging.basicConfig(level=logging.DEBUG) +logging.basicConfig(level=logging.INFO, + format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', + datefmt="%Y-%m-%d %H:%M:%S") diff --git a/healthtools/core.py b/healthtools/core.py index 82eb0ec..652f930 100644 --- a/healthtools/core.py +++ b/healthtools/core.py @@ -63,14 +63,3 @@ def __call__(self, req): req = super(AWS4AuthNotUnicode, self).__call__(req) req.headers = {str(name): value for name, value in req.headers.items()} return req - -def print_error(message): - ''' - Print error messages in the terminal. - ''' - error = "- ERROR: " + message['ERROR'] - source = ("- SOURCE: " + message['SOURCE']) if "SOURCE" in message else "" - error_msg = "- MESSAGE: " + message['MESSAGE'] - msg = "\n".join([error, source, error_msg]) - - log.error(msg) diff --git a/manage.py b/healthtools/manage.py similarity index 100% rename from manage.py rename to healthtools/manage.py diff --git a/healthtools/search/elastic.py b/healthtools/search/elastic.py index 556da0a..29c5d25 100644 --- a/healthtools/search/elastic.py +++ b/healthtools/search/elastic.py @@ -1,5 +1,7 @@ -from healthtools.core import es, es_index, print_error +import logging +from healthtools.core import es, es_index +log = logging.getLogger(__name__) def search(query, doc_type): try: @@ -8,15 +10,11 @@ def search(query, doc_type): body={'query': match_all_text(query)}, doc_type=doc_type ) + hits = result.get('hits', {}) return hits except Exception as err: - error = { - "ERROR": "Elastic Search", - "MESSAGE": str(err) - } - print_error(error) - + log.error("Error fetching data from elastic search \n" + str(err)) def match_all(): diff --git a/healthtools/search/nurses.py b/healthtools/search/nurses.py index 41886d6..38df82b 100644 --- a/healthtools/search/nurses.py +++ b/healthtools/search/nurses.py @@ -1,11 +1,12 @@ import requests +import logging from bs4 import BeautifulSoup -from healthtools.core import print_error NURSING_COUNCIL_URL = 'http://nckenya.com/services/search.php?p=1&s={}' NURSES_FIELDS = ['name', 'licence_no', 'valid_till'] +log = logging.getLogger(__name__) def search(query): results = get_nurses_from_nc_registry(query) @@ -41,8 +42,4 @@ def get_nurses_from_nc_registry(query): return nurses except Exception as err: - error = { - "ERROR": "get_nurses_from_nc_registry()", - "MESSAGE": str(err) - } - print_error(error) + log.error("Error getting nurses from the nursing council url \n" + str(err)) diff --git a/healthtools/search/query.py b/healthtools/search/query.py index b7b2301..abbe49c 100644 --- a/healthtools/search/query.py +++ b/healthtools/search/query.py @@ -1,12 +1,12 @@ +import logging from wit import Wit from nested_lookup import nested_lookup from healthtools.settings import WIT_ACCESS_TOKEN from healthtools.documents import DOCUMENTS, doc_exists -from healthtools.core import print_error - from healthtools.search import elastic, nurses +log = logging.getLogger(__name__) def run_query(query, doc_type=None): @@ -19,7 +19,6 @@ def run_query(query, doc_type=None): else: search_type = 'nurses' return doc_type, search_type - doc_type, search_type = determine_doc_type(query, doc_type) if not doc_type: @@ -56,11 +55,7 @@ def determine_doc_type(query, doc_type=None): for keyword in DOCUMENTS[doc]['keywords']: if query.startswith(keyword + ' '): return doc, DOCUMENTS[doc]['search_type'] - error = { - "ERROR": "doc_type could not be determined from query", - "MESSAGE": 'Query supplied = ' + query - } - print_error(error) + log.error("doc_type could not be determined from query\n Query: " + query) return False, False diff --git a/healthtools/views/search_api.py b/healthtools/views/search_api.py index 82d2930..d26ce8b 100644 --- a/healthtools/views/search_api.py +++ b/healthtools/views/search_api.py @@ -1,9 +1,10 @@ +import logging from flask import Blueprint, request, jsonify from healthtools.search import run_query -from healthtools.core import print_error blueprint = Blueprint('search_api', __name__) +log = logging.getLogger(__name__) @blueprint.route('/search', methods=['GET'], strict_slashes=False) @blueprint.route('/search/', methods=['GET'], strict_slashes=False) @@ -25,11 +26,7 @@ def index(doc_type=None): 'status': 'FAILED', 'msg': '' # TODO: Pass run_query message here. }) - error = { - "ERROR": "index()", - "MESSAGE": str(err) - } - print_error(error) + log.error('Search failed \n' + str(err)) # TODO: Log event here (send to Google Analytics) return response