From 4222526181b738b19d264ff288061a8eee6b1c86 Mon Sep 17 00:00:00 2001 From: sbabayan <34922408+sbabayan@users.noreply.github.com> Date: Fri, 18 Oct 2019 10:24:00 -0700 Subject: [PATCH 01/61] added anvoa to supported pre-deployed models in tabpy (#350) * added anvoa to supported pre-deployed models in tabpy * fixed pep8 issue * fixed md --- docs/tabpy-tools.md | 17 +++++++++++++ tabpy/models/deploy_models.py | 1 - tabpy/models/scripts/ANOVA.py | 25 +++++++++++++++++++ tabpy/models/scripts/PCA.py | 2 -- tabpy/models/scripts/SentimentAnalysis.py | 2 -- tabpy/models/scripts/tTest.py | 2 -- .../test_deploy_model_ssl_off_auth_off.py | 2 +- .../test_deploy_model_ssl_off_auth_on.py | 2 +- .../test_deploy_model_ssl_on_auth_off.py | 2 +- .../test_deploy_model_ssl_on_auth_on.py | 2 +- 10 files changed, 46 insertions(+), 11 deletions(-) create mode 100644 tabpy/models/scripts/ANOVA.py diff --git a/docs/tabpy-tools.md b/docs/tabpy-tools.md index 07ca73c4..e80f7828 100755 --- a/docs/tabpy-tools.md +++ b/docs/tabpy-tools.md @@ -14,6 +14,7 @@ on TabPy server. * [Principal Component Analysis (PCA)](#principal-component-analysis-pca) * [Sentiment Analysis](#sentiment-analysis) * [T-Test](#t-test) + * [ANOVA](#anova) - [Providing Schema Metadata](#providing-schema-metadata) - [Querying an Endpoint](#querying-an-endpoint) - [Evaluating Arbitrary Python Scripts](#evaluating-arbitrary-python-scripts) @@ -318,6 +319,22 @@ The function returns a two-tailed [p-value](https://en.wikipedia.org/wiki/P-valu you may reject or fail to reject the null hypothesis. +### ANOVA + +[Analysis of variance](https://en.wikipedia.org/wiki/Analysis_of_variance) +helps inform if two or more group means within a sample differ. By measuring +the variation between and among groups and computing the resulting F-statistic +we are able to obtain a p-value. While a statistically significant p-value +will inform you that at least 2 of your groups’ means are different from each +other, it will not tell you which of the two groups differ. + +You can call ANOVA from tableau in the following way, + +```python + +tabpy.query(‘anova’, _arg1, _arg2, _arg3)[‘response’] +``` + ## Providing Schema Metadata As soon as you share your deployed functions, you also need to share metadata diff --git a/tabpy/models/deploy_models.py b/tabpy/models/deploy_models.py index 30dda263..a2d6b63c 100644 --- a/tabpy/models/deploy_models.py +++ b/tabpy/models/deploy_models.py @@ -2,7 +2,6 @@ import os import sys import platform -import runpy import subprocess from pathlib import Path from tabpy.models.utils import setup_utils diff --git a/tabpy/models/scripts/ANOVA.py b/tabpy/models/scripts/ANOVA.py new file mode 100644 index 00000000..4bb76bbf --- /dev/null +++ b/tabpy/models/scripts/ANOVA.py @@ -0,0 +1,25 @@ +import scipy.stats as stats +from tabpy.models.utils import setup_utils + + +def anova(_arg1, _arg2, *_argN): + ''' + ANOVA is a statistical hypothesis test that is used to compare + two or more group means for equality.For more information on + the function and how to use it please refer to tabpy-tools.md + ''' + + cols = [_arg1, _arg2] + list(_argN) + for col in cols: + if not isinstance(col[0], (int, float)): + print("values must be numeric") + raise ValueError + f_stat, p_value = stats.f_oneway(_arg1, _arg2, *_argN) + return p_value + + +if __name__ == '__main__': + setup_utils.deploy_model( + 'anova', + anova, + 'Returns the p-value form an ANOVA test') diff --git a/tabpy/models/scripts/PCA.py b/tabpy/models/scripts/PCA.py index 2a2d6b20..f9f5f492 100644 --- a/tabpy/models/scripts/PCA.py +++ b/tabpy/models/scripts/PCA.py @@ -4,8 +4,6 @@ from sklearn.preprocessing import StandardScaler from sklearn.preprocessing import LabelEncoder from sklearn.preprocessing import OneHotEncoder -import sys -from pathlib import Path from tabpy.models.utils import setup_utils diff --git a/tabpy/models/scripts/SentimentAnalysis.py b/tabpy/models/scripts/SentimentAnalysis.py index d3b97e7a..4a978b9f 100644 --- a/tabpy/models/scripts/SentimentAnalysis.py +++ b/tabpy/models/scripts/SentimentAnalysis.py @@ -1,8 +1,6 @@ from textblob import TextBlob import nltk from nltk.sentiment.vader import SentimentIntensityAnalyzer -import sys -from pathlib import Path from tabpy.models.utils import setup_utils diff --git a/tabpy/models/scripts/tTest.py b/tabpy/models/scripts/tTest.py index 9bbc0823..8fffee37 100644 --- a/tabpy/models/scripts/tTest.py +++ b/tabpy/models/scripts/tTest.py @@ -1,6 +1,4 @@ from scipy import stats -import sys -from pathlib import Path from tabpy.models.utils import setup_utils diff --git a/tests/integration/test_deploy_model_ssl_off_auth_off.py b/tests/integration/test_deploy_model_ssl_off_auth_off.py index f5b0749d..01d97490 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_off.py @@ -9,7 +9,7 @@ def test_deploy_ssl_off_auth_off(self): conn = self._get_connection() - models = ['PCA', 'Sentiment%20Analysis', "ttest"] + models = ['PCA', 'Sentiment%20Analysis', "ttest", "anova"] for m in models: conn.request("GET", f'/endpoints/{m}') m_request = conn.getresponse() diff --git a/tests/integration/test_deploy_model_ssl_off_auth_on.py b/tests/integration/test_deploy_model_ssl_off_auth_on.py index 0f09bdc6..7092b4d8 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_on.py @@ -22,7 +22,7 @@ def test_deploy_ssl_off_auth_on(self): conn = self._get_connection() - models = ['PCA', 'Sentiment%20Analysis', "ttest"] + models = ['PCA', 'Sentiment%20Analysis', "ttest", "anova"] for m in models: conn.request("GET", f'/endpoints/{m}', headers=headers) m_request = conn.getresponse() diff --git a/tests/integration/test_deploy_model_ssl_on_auth_off.py b/tests/integration/test_deploy_model_ssl_on_auth_off.py index 8c549b47..584ce648 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_off.py @@ -23,7 +23,7 @@ def test_deploy_ssl_on_auth_off(self): # Do not warn about insecure request requests.packages.urllib3.disable_warnings() - models = ['PCA', 'Sentiment%20Analysis', "ttest"] + models = ['PCA', 'Sentiment%20Analysis', "ttest", "anova"] for m in models: m_response = session.get(url=f'{self._get_transfer_protocol()}://' f'localhost:9004/endpoints/{m}') diff --git a/tests/integration/test_deploy_model_ssl_on_auth_on.py b/tests/integration/test_deploy_model_ssl_on_auth_on.py index 142d6cde..36739252 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_on.py @@ -38,7 +38,7 @@ def test_deploy_ssl_on_auth_on(self): # Do not warn about insecure request requests.packages.urllib3.disable_warnings() - models = ['PCA', 'Sentiment%20Analysis', "ttest"] + models = ['PCA', 'Sentiment%20Analysis', "ttest", "anova"] for m in models: m_response = session.get(url=f'{self._get_transfer_protocol()}://' f'localhost:9004/endpoints/{m}', From 2cfb0a49ba5f41ddad68430b225dc7c562ac3727 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Fri, 18 Oct 2019 17:42:50 -0700 Subject: [PATCH 02/61] Add Ctrl+C handler (#348) * Add Ctrl+C handler * Fix unit tests warnings for genson * Add test to increase code coverage * Add * Change default from 10Mb to 100Mb for request size * Increase code coverage * Increase code coverage * Convert buffer size to int * Add Ctrl+C test * Delete test added to the wrong folder --- CHANGELOG | 7 +++ CONTRIBUTING.md | 12 ++++++ docs/server-config.md | 10 ++++- tabpy/VERSION | 2 +- tabpy/tabpy_server/app/ConfigParameters.py | 1 + tabpy/tabpy_server/app/SettingsParameters.py | 1 + tabpy/tabpy_server/app/app.py | 43 +++++++++++++++---- tabpy/tabpy_server/common/default.conf | 7 ++- .../handlers/management_handler.py | 2 +- .../handlers/query_plane_handler.py | 2 + tabpy/tabpy_server/psws/python_service.py | 4 ++ tabpy/tabpy_tools/rest.py | 5 +-- tabpy/tabpy_tools/schema.py | 6 +-- tests/integration/integ_test_base.py | 23 +++++----- tests/integration/test_app.py | 16 +++++++ .../test_deploy_and_evaluate_model.py | 2 +- .../test_deploy_and_evaluate_model_ssl.py | 5 +++ tests/unit/server_tests/test_config.py | 13 ++++++ 18 files changed, 132 insertions(+), 29 deletions(-) create mode 100755 tests/integration/test_app.py diff --git a/CHANGELOG b/CHANGELOG index 9eb5e61a..33a916f9 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,12 @@ # Changelog +## v0.8.9 + +### Improvements + +- Added Ctrl+C handler +- Added configurable buffer size for HTTP requests + ## v0.8.7 ### Improvements diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0a94bdfd..63518658 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -177,3 +177,15 @@ TabPy package: python setup.py sdist bdist_wheel python -m twine upload dist/* ``` + +To publish test version of the package use the following command: + +```sh +python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* +``` + +To install package from TestPyPi use the command: + +```sh +pip install -i https://test.pypi.org/simple/ tabpy +``` diff --git a/docs/server-config.md b/docs/server-config.md index 39c1ca39..e6e6508e 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -85,6 +85,9 @@ at [`logging.config` documentation page](https://docs.python.org/3.6/library/log not set. - `TABPY_LOG_DETAILS` - when set to `true` additional call information (caller IP, URL, client info, etc.) is logged. Default value - `false`. +- `TABPY_MAX_REQUEST_SIZE_MB` - maximal request size supported by TabPy server + in Megabytes. All requests of exceeding size are rejected. Default value is + 100 Mb. - `TABPY_EVALUATE_TIMEOUT` - script evaluation timeout in seconds. Default value - `30`. @@ -116,10 +119,15 @@ settings._ # end user info if provided. # TABPY_LOG_DETAILS = true +# Limit request size (in Mb) - any request which size exceeds +# specified amount will be rejected by TabPy. +# Default value is 100 Mb. +# TABPY_MAX_REQUEST_SIZE_MB = 100 + # Configure how long a custom script provided to the /evaluate method # will run before throwing a TimeoutError. # The value should be a float representing the timeout time in seconds. -#TABPY_EVALUATE_TIMEOUT = 30 +# TABPY_EVALUATE_TIMEOUT = 30 [loggers] keys=root diff --git a/tabpy/VERSION b/tabpy/VERSION index 35864a97..021abec7 100755 --- a/tabpy/VERSION +++ b/tabpy/VERSION @@ -1 +1 @@ -0.8.7 \ No newline at end of file +0.8.9 \ No newline at end of file diff --git a/tabpy/tabpy_server/app/ConfigParameters.py b/tabpy/tabpy_server/app/ConfigParameters.py index 9ff1a4ad..14abbca2 100644 --- a/tabpy/tabpy_server/app/ConfigParameters.py +++ b/tabpy/tabpy_server/app/ConfigParameters.py @@ -12,4 +12,5 @@ class ConfigParameters: TABPY_PWD_FILE = 'TABPY_PWD_FILE' TABPY_LOG_DETAILS = 'TABPY_LOG_DETAILS' TABPY_STATIC_PATH = 'TABPY_STATIC_PATH' + TABPY_MAX_REQUEST_SIZE_MB = 'TABPY_MAX_REQUEST_SIZE_MB' TABPY_EVALUATE_TIMEOUT = 'TABPY_EVALUATE_TIMEOUT' diff --git a/tabpy/tabpy_server/app/SettingsParameters.py b/tabpy/tabpy_server/app/SettingsParameters.py index 562b6de4..a455fdaa 100755 --- a/tabpy/tabpy_server/app/SettingsParameters.py +++ b/tabpy/tabpy_server/app/SettingsParameters.py @@ -12,4 +12,5 @@ class SettingsParameters: ApiVersions = 'versions' LogRequestContext = 'log_request_context' StaticPath = 'static_path' + MaxRequestSizeInMb = 'max_request_size_in_mb' EvaluateTimeout = 'evaluate_timeout' diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 7297bc9e..9788c96c 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -6,6 +6,7 @@ import multiprocessing import os import shutil +import signal import tabpy.tabpy_server from tabpy.tabpy import __version__ from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters @@ -60,6 +61,9 @@ def __init__(self, config_file=None): def run(self): application = self._create_tornado_web_app() + max_request_size =\ + int(self.settings[SettingsParameters.MaxRequestSizeInMb]) * 1024 * 1024 + logger.info(f'Setting max request size to {max_request_size} bytes') init_model_evaluator( self.settings, @@ -67,25 +71,41 @@ def run(self): self.python_service) protocol = self.settings[SettingsParameters.TransferProtocol] - if protocol == 'http': - application.listen(self.settings[SettingsParameters.Port]) - elif protocol == 'https': - application.listen(self.settings[SettingsParameters.Port], - ssl_options={ + ssl_options = None + if protocol == 'https': + ssl_options = { 'certfile': self.settings[SettingsParameters.CertificateFile], 'keyfile': self.settings[SettingsParameters.KeyFile] - }) - else: + } + elif protocol != 'http': msg = f'Unsupported transfer protocol {protocol}.' logger.critical(msg) raise RuntimeError(msg) + application.listen( + self.settings[SettingsParameters.Port], + ssl_options=ssl_options, + max_buffer_size=max_request_size, + max_body_size=max_request_size) + logger.info( 'Web service listening on port ' f'{str(self.settings[SettingsParameters.Port])}') tornado.ioloop.IOLoop.instance().start() def _create_tornado_web_app(self): + class TabPyTornadoApp(tornado.web.Application): + is_closing = False + + def signal_handler(self, signal, frame): + logger.critical(f'Exiting on signal {signal}...') + self.is_closing = True + + def try_exit(self): + if self.is_closing: + tornado.ioloop.IOLoop.instance().stop() + logger.info('Shutting down TabPy...') + logger.info('Initializing TabPy...') tornado.ioloop.IOLoop.instance().run_sync( lambda: init_ps_server(self.settings, self.tabpy_state)) @@ -95,7 +115,7 @@ def _create_tornado_web_app(self): max_workers=multiprocessing.cpu_count()) # initialize Tornado application - application = tornado.web.Application([ + application = TabPyTornadoApp([ # skip MainHandler to use StaticFileHandler .* page requests and # default to index.html # (r"/", MainHandler), @@ -121,6 +141,9 @@ def _create_tornado_web_app(self): default_filename="index.html")), ], debug=False, **self.settings) + signal.signal(signal.SIGINT, application.signal_handler) + tornado.ioloop.PeriodicCallback(application.try_exit, 500).start() + return application @staticmethod @@ -303,6 +326,10 @@ def set_parameter(settings_key, else 'disabled' logger.info(f'Call context logging is {call_context_state}') + set_parameter(SettingsParameters.MaxRequestSizeInMb, + ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB, + default_val=100) + def _validate_transfer_protocol_settings(self): if SettingsParameters.TransferProtocol not in self.settings: msg = 'Missing transfer protocol information.' diff --git a/tabpy/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf index 52786491..ee02453d 100755 --- a/tabpy/tabpy_server/common/default.conf +++ b/tabpy/tabpy_server/common/default.conf @@ -20,10 +20,15 @@ # end user info if provided. # TABPY_LOG_DETAILS = true +# Limit request size (in Mb) - any request which size exceeds +# specified amount will be rejected by TabPy. +# Default value is 100 Mb. +# TABPY_MAX_REQUEST_SIZE_MB = 100 + # Configure how long a custom script provided to the /evaluate method # will run before throwing a TimeoutError. # The value should be a float representing the timeout time in seconds. -#TABPY_EVALUATE_TIMEOUT = 30 +# TABPY_EVALUATE_TIMEOUT = 30 [loggers] keys=root diff --git a/tabpy/tabpy_server/handlers/management_handler.py b/tabpy/tabpy_server/handlers/management_handler.py index 805d3e51..e7a0b3c0 100644 --- a/tabpy/tabpy_server/handlers/management_handler.py +++ b/tabpy/tabpy_server/handlers/management_handler.py @@ -94,7 +94,7 @@ def _add_or_update_endpoint(self, action, name, version, request_data): self.settings[SettingsParameters.StateFilePath], name, version) self.logger.log(logging.DEBUG, f'Checking source path {src_path}...') - _path_checker = _compile(r'^[\\\:a-zA-Z0-9-_~\s/\.]+$') + _path_checker = _compile(r'^[\\\:a-zA-Z0-9-_~\s/\.\(\)]+$') # copy from staging if src_path: if not isinstance(request_data['src_path'], str): diff --git a/tabpy/tabpy_server/handlers/query_plane_handler.py b/tabpy/tabpy_server/handlers/query_plane_handler.py index 70774626..61657a39 100644 --- a/tabpy/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy/tabpy_server/handlers/query_plane_handler.py @@ -135,6 +135,7 @@ def _process_query(self, endpoint_name, start): # Sanitize input data data = self._sanitize_request_data(json.loads(request_json)) except Exception as e: + self.logger.log(logging.ERROR, str(e)) err_msg = format_exception(e, "Invalid Input Data") self.error_out(400, err_msg) return @@ -177,6 +178,7 @@ def _process_query(self, endpoint_name, start): return except Exception as e: + self.logger.log(logging.ERROR, str(e)) err_msg = format_exception(e, 'process query') self.error_out(500, 'Error processing query', info=err_msg) return diff --git a/tabpy/tabpy_server/psws/python_service.py b/tabpy/tabpy_server/psws/python_service.py index 3fd9aa96..768b113e 100644 --- a/tabpy/tabpy_server/psws/python_service.py +++ b/tabpy/tabpy_server/psws/python_service.py @@ -42,6 +42,7 @@ def manage_request(self, msg): logger.debug(f'Returning response {response}') return response except Exception as e: + logger.exception(e) msg = e if hasattr(e, 'message'): msg = e.message @@ -90,6 +91,7 @@ def _load_object(self, object_uri, object_url, object_version, is_update, 'status': 'LoadSuccessful', 'last_error': None} except Exception as e: + logger.exception(e) logger.error(f'Unable to load QueryObject: path={object_url}, ' f'error={str(e)}') @@ -132,6 +134,7 @@ def load_object(self, object_uri, object_url, object_version, is_update, object_uri, object_url, object_version, is_update, object_type) except Exception as e: + logger.exception(e) logger.error(f'Unable to load QueryObject: path={object_url}, ' f'error={str(e)}') @@ -226,6 +229,7 @@ def query(self, object_uri, params, uid): else: return UnknownURI(object_uri) except Exception as e: + logger.exception(e) err_msg = format_exception(e, '/query') logger.error(err_msg) return QueryFailed(uri=object_uri, error=err_msg) diff --git a/tabpy/tabpy_tools/rest.py b/tabpy/tabpy_tools/rest.py index 7189ff49..37db25d4 100755 --- a/tabpy/tabpy_tools/rest.py +++ b/tabpy/tabpy_tools/rest.py @@ -1,12 +1,11 @@ import abc +from collections.abc import MutableMapping import logging import requests from requests.auth import HTTPBasicAuth from re import compile import json as json -from collections import MutableMapping as _MutableMapping - logger = logging.getLogger(__name__) @@ -290,7 +289,7 @@ def __init__(self, name, bases, dict): self.__rest__.add(k) -class RESTObject(_MutableMapping, metaclass=_RESTMetaclass): +class RESTObject(MutableMapping, metaclass=_RESTMetaclass): """A base class that has methods generally useful for interacting with REST objects. The attributes are accessible either as dict keys or as attributes. The object also behaves like a dict, even replicating the diff --git a/tabpy/tabpy_tools/schema.py b/tabpy/tabpy_tools/schema.py index 080d3529..6fc32556 100755 --- a/tabpy/tabpy_tools/schema.py +++ b/tabpy/tabpy_tools/schema.py @@ -1,5 +1,5 @@ import logging -import genson as _genson +import genson import jsonschema @@ -12,9 +12,9 @@ def _generate_schema_from_example_and_description(input, description): to the example in json-schema.org. The description given by the users is then added to the schema. ''' - s = _genson.Schema() + s = genson.SchemaBuilder(None) s.add_object(input) - input_schema = s.to_dict() + input_schema = s.to_schema() if description is not None: if 'properties' in input_schema: diff --git a/tests/integration/integ_test_base.py b/tests/integration/integ_test_base.py index 13305642..57e246e6 100755 --- a/tests/integration/integ_test_base.py +++ b/tests/integration/integ_test_base.py @@ -227,20 +227,19 @@ def setUp(self): with open(self.tmp_dir + '/output.txt', 'w') as outfile: cmd = ['tabpy', '--config=' + self.config_file_name] - coverage.process_startup() + preexec_fn = None if platform.system() == 'Windows': self.py = 'python' - self.process = subprocess.Popen( - cmd, - stdout=outfile, - stderr=outfile) else: self.py = 'python3' - self.process = subprocess.Popen( - cmd, - preexec_fn=os.setsid, - stdout=outfile, - stderr=outfile) + preexec_fn = os.setsid + + coverage.process_startup() + self.process = subprocess.Popen( + cmd, + preexec_fn=preexec_fn, + stdout=outfile, + stderr=outfile) # give the app some time to start up... time.sleep(5) @@ -299,3 +298,7 @@ def deploy_models(self, username: str, password: str): input=input_string.encode('utf-8'), stdout=outfile, stderr=outfile) + + def _get_process(self): + return self.process + diff --git a/tests/integration/test_app.py b/tests/integration/test_app.py new file mode 100755 index 00000000..125a703e --- /dev/null +++ b/tests/integration/test_app.py @@ -0,0 +1,16 @@ +import integ_test_base +import os +import signal +import unittest + +class TestApp(integ_test_base.IntegTestBase): + def test_ctrl_c(self): + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + # self.set_delete_temp_folder(False) + + process = self._get_process() + os.kill(process.pid, signal.SIGINT) + + diff --git a/tests/integration/test_deploy_and_evaluate_model.py b/tests/integration/test_deploy_and_evaluate_model.py index 3a2e80f5..30274ebe 100644 --- a/tests/integration/test_deploy_and_evaluate_model.py +++ b/tests/integration/test_deploy_and_evaluate_model.py @@ -14,7 +14,7 @@ def test_deploy_and_evaluate_model(self): # Uncomment the following line to preserve # test case output and other files (config, state, ect.) # in system temp folder. - self.set_delete_temp_folder(False) + # self.set_delete_temp_folder(False) self.deploy_models(self._get_username(), self._get_password()) diff --git a/tests/integration/test_deploy_and_evaluate_model_ssl.py b/tests/integration/test_deploy_and_evaluate_model_ssl.py index 09a68feb..dc5f7956 100755 --- a/tests/integration/test_deploy_and_evaluate_model_ssl.py +++ b/tests/integration/test_deploy_and_evaluate_model_ssl.py @@ -18,6 +18,11 @@ def _get_key_file_name(self) -> str: return './tests/integration/resources/2019_04_24_to_3018_08_25.key' def test_deploy_and_evaluate_model_ssl(self): + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + self.set_delete_temp_folder(False) + self.deploy_models(self._get_username(), self._get_password()) payload = ( diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 2ecffb7e..adacd8e5 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -10,6 +10,19 @@ class TestConfigEnvironmentCalls(unittest.TestCase): + def test_config_file_does_not_exist(self): + app = TabPyApp('/folder_does_not_exit/file_does_not_exist.conf') + + self.assertEqual(app.settings['port'], 9004) + self.assertEqual(app.settings['server_version'], + open('tabpy/VERSION').read().strip()) + self.assertEqual(app.settings['transfer_protocol'], 'http') + self.assertTrue('certificate_file' not in app.settings) + self.assertTrue('key_file' not in app.settings) + self.assertEqual(app.settings['log_request_context'], False) + self.assertEqual(app.settings['evaluate_timeout'], 30) + + @patch('tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace(config=None)) @patch('tabpy.tabpy_server.app.app.TabPyState') From b899aa64daa3f519533716c8eebc3c9f96119628 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Fri, 18 Oct 2019 17:48:22 -0700 Subject: [PATCH 03/61] Update CHANGELOG --- CHANGELOG | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG b/CHANGELOG index 33a916f9..dcd61df3 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,6 +6,7 @@ - Added Ctrl+C handler - Added configurable buffer size for HTTP requests +- Added anvoa to supported pre-deployed models in tabpy ## v0.8.7 From deeefc19de59cea96b2e0cd52346578b9152ed1e Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Sat, 19 Oct 2019 18:17:11 -0700 Subject: [PATCH 04/61] Update test_app.py --- tests/integration/test_app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/test_app.py b/tests/integration/test_app.py index 125a703e..ed5b2120 100755 --- a/tests/integration/test_app.py +++ b/tests/integration/test_app.py @@ -3,6 +3,7 @@ import signal import unittest + class TestApp(integ_test_base.IntegTestBase): def test_ctrl_c(self): # Uncomment the following line to preserve @@ -12,5 +13,3 @@ def test_ctrl_c(self): process = self._get_process() os.kill(process.pid, signal.SIGINT) - - From 8dad7654f2c0ab10f71fc8b2969ef680efa5fe50 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Mon, 21 Oct 2019 10:13:15 -0700 Subject: [PATCH 05/61] Remove dead code --- tabpy/tabpy_server/app/app.py | 3 +- tabpy/tabpy_server/management/util.py | 30 ------------------- tests/integration/integ_test_base.py | 1 - tests/integration/test_app.py | 16 ---------- .../test_deploy_and_evaluate_model_ssl.py | 2 +- .../test_deploy_model_ssl_off_auth_off.py | 5 ++++ tests/unit/server_tests/test_config.py | 1 - 7 files changed, 8 insertions(+), 50 deletions(-) delete mode 100755 tests/integration/test_app.py diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 9788c96c..fd331c4b 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -62,7 +62,8 @@ def __init__(self, config_file=None): def run(self): application = self._create_tornado_web_app() max_request_size =\ - int(self.settings[SettingsParameters.MaxRequestSizeInMb]) * 1024 * 1024 + int(self.settings[SettingsParameters.MaxRequestSizeInMb]) *\ + 1024 * 1024 logger.info(f'Setting max request size to {max_request_size} bytes') init_model_evaluator( diff --git a/tabpy/tabpy_server/management/util.py b/tabpy/tabpy_server/management/util.py index 13d1eae0..2cc5efa2 100644 --- a/tabpy/tabpy_server/management/util.py +++ b/tabpy/tabpy_server/management/util.py @@ -48,33 +48,3 @@ def _get_state_from_file(state_path, logger=logging.getLogger(__name__)): _ZERO = timedelta(0) - - -class _UTC(tzinfo): - """ - A UTC datetime.tzinfo class modeled after the pytz library. It includes a - __reduce__ method for pickling, - """ - - def fromutc(self, dt): - if dt.tzinfo is None: - return self.localize(dt) - return super(_UTC, self).fromutc(dt) - - def utcoffset(self, dt): - return _ZERO - - def tzname(self, dt): - return "UTC" - - def dst(self, dt): - return _ZERO - - def __reduce__(self): - return _UTC, () - - def __repr__(self): - return "" - - def __str__(self): - return "UTC" diff --git a/tests/integration/integ_test_base.py b/tests/integration/integ_test_base.py index 57e246e6..5ed8aa60 100755 --- a/tests/integration/integ_test_base.py +++ b/tests/integration/integ_test_base.py @@ -301,4 +301,3 @@ def deploy_models(self, username: str, password: str): def _get_process(self): return self.process - diff --git a/tests/integration/test_app.py b/tests/integration/test_app.py deleted file mode 100755 index 125a703e..00000000 --- a/tests/integration/test_app.py +++ /dev/null @@ -1,16 +0,0 @@ -import integ_test_base -import os -import signal -import unittest - -class TestApp(integ_test_base.IntegTestBase): - def test_ctrl_c(self): - # Uncomment the following line to preserve - # test case output and other files (config, state, ect.) - # in system temp folder. - # self.set_delete_temp_folder(False) - - process = self._get_process() - os.kill(process.pid, signal.SIGINT) - - diff --git a/tests/integration/test_deploy_and_evaluate_model_ssl.py b/tests/integration/test_deploy_and_evaluate_model_ssl.py index dc5f7956..bb928a75 100755 --- a/tests/integration/test_deploy_and_evaluate_model_ssl.py +++ b/tests/integration/test_deploy_and_evaluate_model_ssl.py @@ -21,7 +21,7 @@ def test_deploy_and_evaluate_model_ssl(self): # Uncomment the following line to preserve # test case output and other files (config, state, ect.) # in system temp folder. - self.set_delete_temp_folder(False) + # self.set_delete_temp_folder(False) self.deploy_models(self._get_username(), self._get_password()) diff --git a/tests/integration/test_deploy_model_ssl_off_auth_off.py b/tests/integration/test_deploy_model_ssl_off_auth_off.py index 01d97490..1aa0d097 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_off.py @@ -5,6 +5,11 @@ class TestDeployModelSSLOffAuthOff(integ_test_base.IntegTestBase): def test_deploy_ssl_off_auth_off(self): + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + # self.set_delete_temp_folder(False) + self.deploy_models(self._get_username(), self._get_password()) conn = self._get_connection() diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index adacd8e5..1d285305 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -22,7 +22,6 @@ def test_config_file_does_not_exist(self): self.assertEqual(app.settings['log_request_context'], False) self.assertEqual(app.settings['evaluate_timeout'], 30) - @patch('tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments', return_value=Namespace(config=None)) @patch('tabpy.tabpy_server.app.app.TabPyState') From 1413c8d0b0e80f2c90a1c64d751a723f276fd6fb Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Mon, 21 Oct 2019 12:15:05 -0700 Subject: [PATCH 06/61] Don't count coverage for multiline expressions --- .coveragerc | 1 + 1 file changed, 1 insertion(+) diff --git a/.coveragerc b/.coveragerc index 228ef0f8..6a64b4d1 100755 --- a/.coveragerc +++ b/.coveragerc @@ -2,6 +2,7 @@ # Exclude lines that match patterns from coverage report. exclude_lines = if __name__ == .__main__.: + \\$ # Only show one number after decimal point in report. precision = 1 From 98f7026e08b89c3df4c28c895fef1a5894bb4642 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Mon, 21 Oct 2019 13:43:02 -0700 Subject: [PATCH 07/61] Add test case for invalid protocol --- tabpy/models/scripts/ANOVA.py | 2 +- tabpy/tabpy_server/app/app.py | 2 +- tabpy/tabpy_server/management/util.py | 2 -- tabpy/tabpy_tools/rest.py | 2 +- tests/unit/server_tests/test_config.py | 8 ++++++++ 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tabpy/models/scripts/ANOVA.py b/tabpy/models/scripts/ANOVA.py index 4bb76bbf..b151b086 100644 --- a/tabpy/models/scripts/ANOVA.py +++ b/tabpy/models/scripts/ANOVA.py @@ -14,7 +14,7 @@ def anova(_arg1, _arg2, *_argN): if not isinstance(col[0], (int, float)): print("values must be numeric") raise ValueError - f_stat, p_value = stats.f_oneway(_arg1, _arg2, *_argN) + _, p_value = stats.f_oneway(_arg1, _arg2, *_argN) return p_value diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index fd331c4b..7ced39c0 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -98,7 +98,7 @@ def _create_tornado_web_app(self): class TabPyTornadoApp(tornado.web.Application): is_closing = False - def signal_handler(self, signal, frame): + def signal_handler(self, signal): logger.critical(f'Exiting on signal {signal}...') self.is_closing = True diff --git a/tabpy/tabpy_server/management/util.py b/tabpy/tabpy_server/management/util.py index 2cc5efa2..7bc21244 100644 --- a/tabpy/tabpy_server/management/util.py +++ b/tabpy/tabpy_server/management/util.py @@ -46,5 +46,3 @@ def _get_state_from_file(state_path, logger=logging.getLogger(__name__)): return config - -_ZERO = timedelta(0) diff --git a/tabpy/tabpy_tools/rest.py b/tabpy/tabpy_tools/rest.py index 37db25d4..9446708c 100755 --- a/tabpy/tabpy_tools/rest.py +++ b/tabpy/tabpy_tools/rest.py @@ -245,7 +245,7 @@ def __init__(self, type, from_json=lambda x: x, to_json=lambda x: x, self.from_json = from_json self.to_json = to_json - def __get__(self, instance, owner): + def __get__(self, instance, _): if instance: try: return getattr(instance, self.name) diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 1d285305..d665657b 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -178,6 +178,14 @@ def tearDown(self): os.remove(self.fp.name) self.fp = None + def test_invalid_protocol(self): + self.fp.write("[TabPy]\n" + "TABPY_TRANSFER_PROTOCOL = gopher") + self.fp.close() + + self.assertTabPyAppRaisesRuntimeError( + 'Unsupported transfer protocol: gopher') + def test_http(self): self.fp.write("[TabPy]\n" "TABPY_TRANSFER_PROTOCOL = http") From 0e730b11d9e314962c1501ddc96bb3848b78090c Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Mon, 21 Oct 2019 15:12:31 -0700 Subject: [PATCH 08/61] Add test case for _check_endpoint_name --- tabpy/tabpy_server/app/app.py | 3 +-- tests/unit/tools_tests/test_client.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 7ced39c0..2df69620 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -147,8 +147,7 @@ def try_exit(self): return application - @staticmethod - def _parse_cli_arguments(): + def _parse_cli_arguments(self): ''' Parse command line arguments. Expected arguments: * --config: string diff --git a/tests/unit/tools_tests/test_client.py b/tests/unit/tools_tests/test_client.py index 670069d0..62ffccb0 100644 --- a/tests/unit/tools_tests/test_client.py +++ b/tests/unit/tools_tests/test_client.py @@ -2,6 +2,7 @@ from unittest.mock import Mock from tabpy.tabpy_tools.client import Client +from tabpy.tabpy_tools.client import _check_endpoint_name class TestClient(unittest.TestCase): @@ -82,3 +83,12 @@ def test_set_credentials(self): self.client._service.set_credentials.assert_called_once_with( username, password) + + def test_check_invalid_endpoint_name(self): + endpoint_name = 'Invalid:model:@name' + with self.assertRaises(ValueError) as err: + _check_endpoint_name(endpoint_name) + + self.assertEqual(err.exception.args[0], + f'endpoint name {endpoint_name } can only contain: ' + 'a-z, A-Z, 0-9, underscore, hyphens and spaces.') From ebd1969234e73d5df87b4373ea9a810f7d83682f Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Mon, 21 Oct 2019 15:29:19 -0700 Subject: [PATCH 09/61] Remove dead code --- tabpy/tabpy_tools/client.py | 132 ------------------------------------ 1 file changed, 132 deletions(-) diff --git a/tabpy/tabpy_tools/client.py b/tabpy/tabpy_tools/client.py index d6a5a175..758397cb 100755 --- a/tabpy/tabpy_tools/client.py +++ b/tabpy/tabpy_tools/client.py @@ -97,21 +97,6 @@ def __repr__(self): ' object at ' + hex(id(self)) + ' connected to ' + repr(self._endpoint) + ">") - def get_info(self): - """Returns a dict containing information about the service. - - Returns - ------- - dict - Keys are: - * name: The name of the service - * creation_time: The creation time in seconds since 1970-01-01 - * description: Description of the service - * server_version: The version of the service used - * state_path: Where the state file is stored. - """ - return self._service.get_info() - def get_status(self): ''' Gets the status of the deployed endpoints. @@ -210,57 +195,6 @@ def _get_endpoint_upload_destination(self): """Returns the endpoint upload destination.""" return self._service.get_endpoint_upload_destination()['path'] - def alias(self, alias, existing_endpoint_name, description=None): - ''' - Create a new endpoint to redirect to an existing endpoint, or update an - existing alias to point to a different existing endpoint. - - Parameters - ---------- - alias : str - The new endpoint name or an existing alias endpoint name. - - existing_endpoint_name : str - A name of an existing endpoint to redirect the alias to. - - description : str, optional - A description for the alias. - ''' - # check for invalid PO names - _check_endpoint_name(alias) - - if not description: - description = f'Alias for {existing_endpoint_name}' - - if existing_endpoint_name not in self.get_endpoints(): - raise ValueError( - f'Endpoint "{existing_endpoint_name}" does not exist.') - - # Can only overwrite existing alias - existing_endpoint = self.get_endpoints().get(alias) - endpoint = AliasEndpoint( - name=alias, - type='alias', - description=description, - target=existing_endpoint_name, - cache_state='disabled', - version=1, - ) - - if existing_endpoint: - if existing_endpoint.type != 'alias': - raise RuntimeError( - f'Name "{alias}" is already in use by another ' - 'endpoint.') - - endpoint.version = existing_endpoint.version + 1 - - self._service.set_endpoint(endpoint) - else: - self._service.add_endpoint(endpoint) - - self._wait_for_endpoint_deployment(alias, endpoint.version) - def deploy(self, name, obj, description='', schema=None, override=False): @@ -447,72 +381,6 @@ def _wait_for_endpoint_deployment(self, logger.info(f'Sleeping {interval}...') time.sleep(interval) - def remove(self, name): - ''' - Remove the endpoint that has the specified name. - - Parameters - ---------- - name : str - The name of the endpoint to be removed. - - Notes - ----- - This could fail if the endpoint does not exist, or if the endpoint is - in use by an alias. To check all endpoints - that are depending on this endpoint, use `get_endpoint_dependencies`. - - See Also - -------- - deploy, get_endpoint_dependencies - ''' - self._service.remove_endpoint(name) - - # Wait for the endpoint to be removed - while name in self.get_endpoints(): - time.sleep(1.0) - - def get_endpoint_dependencies(self, endpoint_name=None): - ''' - Get all endpoints that depend on the given endpoint. The only - dependency that is recorded is aliases on the endpoint they refer to. - This will not return internal dependencies, as when you have an - endpoint that calls another endpoint from within its code body. - - Parameters - ---------- - endpoint_name : str, optional - The name of the endpoint to find dependent endpoints. If not given, - find all dependent endpoints for all endpoints. - - Returns - ------- - dependent endpoints : dict - If endpoint_name is given, returns a list of endpoint names that - depend on the given endpoint. - - If endpoint_name is not given, returns a dictionary where key is - the endpoint name and value is a set of endpoints that depend on - the endpoint specified by the key. - ''' - endpoints = self.get_endpoints() - - def get_dependencies(endpoint): - result = set() - for d in endpoints[endpoint].dependencies: - result.update([d]) - result.update(get_dependencies(d)) - return result - - if endpoint_name: - return get_dependencies(endpoint_name) - - else: - return { - endpoint: get_dependencies(endpoint) - for endpoint in endpoints - } - def set_credentials(self, username, password): ''' Set credentials for all the TabPy client-server communication From 146d8aab723a5636d480114d1e56358fa10bba59 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Wed, 27 Nov 2019 13:06:55 -0800 Subject: [PATCH 10/61] Fix vulnerabilities found by LGTM (#361) * Fix vulnerabilities found by LGTM * Fix test failures --- tabpy/models/deploy_models.py | 16 +++++++------- tabpy/models/utils/setup_utils.py | 1 - tabpy/tabpy.py | 4 ++-- tabpy/tabpy_server/app/ConfigParameters.py | 1 - tabpy/tabpy_server/common/messages.py | 3 +-- .../tabpy_server/handlers/endpoint_handler.py | 2 -- .../handlers/management_handler.py | 2 -- .../handlers/query_plane_handler.py | 1 - .../handlers/upload_destination_handler.py | 1 - tabpy/tabpy_server/handlers/util.py | 2 -- tabpy/tabpy_server/management/state.py | 2 -- tabpy/tabpy_server/psws/callbacks.py | 1 - tabpy/tabpy_server/psws/python_service.py | 3 --- tabpy/tabpy_tools/client.py | 21 +++++-------------- tabpy/utils/user_management.py | 2 -- 15 files changed, 15 insertions(+), 47 deletions(-) diff --git a/tabpy/models/deploy_models.py b/tabpy/models/deploy_models.py index a2d6b63c..9130474e 100644 --- a/tabpy/models/deploy_models.py +++ b/tabpy/models/deploy_models.py @@ -1,23 +1,21 @@ -import pip import os -import sys +from pathlib import Path +import pip import platform import subprocess -from pathlib import Path +import sys from tabpy.models.utils import setup_utils -# pip 10.0 introduced a breaking change that moves the location of main -try: - from pip import main -except ImportError: - from pip._internal import main - def install_dependencies(packages): pip_arg = ['install'] + packages + ['--no-cache-dir'] + + # pip 10.0 introduced a breaking change that moves the location of main if hasattr(pip, 'main'): + from pip import main pip.main(pip_arg) else: + from pip._internal import main pip._internal.main(pip_arg) diff --git a/tabpy/models/utils/setup_utils.py b/tabpy/models/utils/setup_utils.py index 65468153..a311d462 100644 --- a/tabpy/models/utils/setup_utils.py +++ b/tabpy/models/utils/setup_utils.py @@ -1,7 +1,6 @@ import configparser import getpass import os -from pathlib import Path import sys from tabpy.tabpy_tools.client import Client diff --git a/tabpy/tabpy.py b/tabpy/tabpy.py index b03f16cf..807d4d64 100755 --- a/tabpy/tabpy.py +++ b/tabpy/tabpy.py @@ -14,11 +14,11 @@ def read_version(): import tabpy pkg_path = os.path.dirname(tabpy.__file__) ver_file_path = os.path.join(pkg_path, 'VERSION') + ver = f'Version Unknown, (file {ver_file_path} not found)' + if Path(ver_file_path).exists(): with open(ver_file_path) as f: ver = f.read().strip() - else: - ver = f'Version Unknown, (file {ver_file_path} not found)' return ver diff --git a/tabpy/tabpy_server/app/ConfigParameters.py b/tabpy/tabpy_server/app/ConfigParameters.py index 14abbca2..9581d49d 100644 --- a/tabpy/tabpy_server/app/ConfigParameters.py +++ b/tabpy/tabpy_server/app/ConfigParameters.py @@ -2,7 +2,6 @@ class ConfigParameters: ''' Configuration settings names ''' - TABPY_PWD_FILE = 'TABPY_PWD_FILE' TABPY_PORT = 'TABPY_PORT' TABPY_QUERY_OBJECT_PATH = 'TABPY_QUERY_OBJECT_PATH' TABPY_STATE_PATH = 'TABPY_STATE_PATH' diff --git a/tabpy/tabpy_server/common/messages.py b/tabpy/tabpy_server/common/messages.py index 5e1354ec..e7952425 100644 --- a/tabpy/tabpy_server/common/messages.py +++ b/tabpy/tabpy_server/common/messages.py @@ -1,5 +1,4 @@ import abc -from abc import ABCMeta from collections import namedtuple import json @@ -16,7 +15,7 @@ class Msg(object): operator (*) that we inherit from namedtuple is also convenient. We empty __slots__ to avoid unnecessary overhead. """ - __metaclass__ = ABCMeta + __metaclass__ = abc.ABCMeta @abc.abstractmethod def for_json(self): diff --git a/tabpy/tabpy_server/handlers/endpoint_handler.py b/tabpy/tabpy_server/handlers/endpoint_handler.py index 20b6b334..6e85cebd 100644 --- a/tabpy/tabpy_server/handlers/endpoint_handler.py +++ b/tabpy/tabpy_server/handlers/endpoint_handler.py @@ -6,7 +6,6 @@ at endpoints_handler.py ''' -import concurrent import json import logging import shutil @@ -16,7 +15,6 @@ from tabpy.tabpy_server.management.state import get_query_object_path from tabpy.tabpy_server.psws.callbacks import on_state_change from tornado import gen -import tornado.web class EndpointHandler(ManagementHandler): diff --git a/tabpy/tabpy_server/handlers/management_handler.py b/tabpy/tabpy_server/handlers/management_handler.py index e7a0b3c0..0f911c43 100644 --- a/tabpy/tabpy_server/handlers/management_handler.py +++ b/tabpy/tabpy_server/handlers/management_handler.py @@ -1,7 +1,5 @@ -import concurrent import logging import os -import sys import shutil from re import compile as _compile from uuid import uuid4 as random_uuid diff --git a/tabpy/tabpy_server/handlers/query_plane_handler.py b/tabpy/tabpy_server/handlers/query_plane_handler.py index 61657a39..dbad5d25 100644 --- a/tabpy/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy/tabpy_server/handlers/query_plane_handler.py @@ -8,7 +8,6 @@ import json from tabpy.tabpy_server.common.util import format_exception import urllib -import tornado.web from tornado import gen diff --git a/tabpy/tabpy_server/handlers/upload_destination_handler.py b/tabpy/tabpy_server/handlers/upload_destination_handler.py index 5211b1e6..6f662476 100644 --- a/tabpy/tabpy_server/handlers/upload_destination_handler.py +++ b/tabpy/tabpy_server/handlers/upload_destination_handler.py @@ -1,4 +1,3 @@ -import logging from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters from tabpy.tabpy_server.handlers import ManagementHandler import os diff --git a/tabpy/tabpy_server/handlers/util.py b/tabpy/tabpy_server/handlers/util.py index e835d7fc..a1747c6c 100755 --- a/tabpy/tabpy_server/handlers/util.py +++ b/tabpy/tabpy_server/handlers/util.py @@ -1,7 +1,5 @@ -import base64 import binascii from hashlib import pbkdf2_hmac -from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters def hash_password(username, pwd): diff --git a/tabpy/tabpy_server/management/state.py b/tabpy/tabpy_server/management/state.py index c1353ece..3ca66f3c 100644 --- a/tabpy/tabpy_server/management/state.py +++ b/tabpy/tabpy_server/management/state.py @@ -4,7 +4,6 @@ from configparser import ConfigParser import json import logging -import sys from tabpy.tabpy_server.management.util import write_state_config from threading import Lock from time import time @@ -470,7 +469,6 @@ def get_access_control_allow_origin(self): 'Service Info', 'Access-Control-Allow-Origin') except Exception as e: logger.error(e) - pass return _cors_origin def get_access_control_allow_headers(self): diff --git a/tabpy/tabpy_server/psws/callbacks.py b/tabpy/tabpy_server/psws/callbacks.py index d6c38b4f..83e55cef 100644 --- a/tabpy/tabpy_server/psws/callbacks.py +++ b/tabpy/tabpy_server/psws/callbacks.py @@ -126,7 +126,6 @@ def _get_latest_service_state(settings, if diff: changes['endpoints'] = diff - tabpy_state = new_ps_state return (True, changes) diff --git a/tabpy/tabpy_server/psws/python_service.py b/tabpy/tabpy_server/psws/python_service.py index 768b113e..8c51a124 100644 --- a/tabpy/tabpy_server/psws/python_service.py +++ b/tabpy/tabpy_server/psws/python_service.py @@ -1,8 +1,5 @@ import concurrent.futures import logging -import sys - - from tabpy.tabpy_tools.query_object import QueryObject from tabpy.tabpy_server.common.util import format_exception from tabpy.tabpy_server.common.messages import ( diff --git a/tabpy/tabpy_tools/client.py b/tabpy/tabpy_tools/client.py index 758397cb..f7f4eeec 100755 --- a/tabpy/tabpy_tools/client.py +++ b/tabpy/tabpy_tools/client.py @@ -1,21 +1,9 @@ from re import compile import time -import sys import requests - -from .rest import ( - RequestsNetworkWrapper, - ServiceClient -) - -from .rest_client import ( - RESTServiceClient, - Endpoint, - AliasEndpoint -) - +from .rest import (RequestsNetworkWrapper, ServiceClient) +from .rest_client import (RESTServiceClient, Endpoint) from .custom_query_object import CustomQueryObject - import os import logging @@ -253,7 +241,7 @@ def deploy(self, self._wait_for_endpoint_deployment(obj['name'], obj['version']) - def _gen_endpoint(self, name, obj, description, version=1, schema=[]): + def _gen_endpoint(self, name, obj, description, version=1, schema=None): '''Generates an endpoint dict. Parameters @@ -311,6 +299,7 @@ def _gen_endpoint(self, name, obj, description, version=1, schema=[]): description=description, ) + _schema = schema if schema is not None else [] return { 'name': name, 'version': version, @@ -321,7 +310,7 @@ def _gen_endpoint(self, name, obj, description, version=1, schema=[]): 'methods': endpoint_object.get_methods(), 'required_files': [], 'required_packages': [], - 'schema': schema + 'schema': _schema } def _upload_endpoint(self, obj): diff --git a/tabpy/utils/user_management.py b/tabpy/utils/user_management.py index 774af11c..c86b15b8 100755 --- a/tabpy/utils/user_management.py +++ b/tabpy/utils/user_management.py @@ -4,9 +4,7 @@ from argparse import ArgumentParser import logging -import os import secrets -import sys from tabpy.tabpy_server.app.util import parse_pwd_file from tabpy.tabpy_server.handlers.util import hash_password From 06572be8af8baf06e27ef0a3935b4c921259d6d3 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Fri, 20 Dec 2019 15:34:14 -0800 Subject: [PATCH 11/61] Dev improvements (#384) * Fix flake8 warnings * Merge from master * Fix pycodestyle * Fix more flake8 warnings * Fix tests to pass again * Create test_coveralls_codestyle.yml (#382) * Use github actions --- .../workflows/test_coveralls_codestyle.yml | 108 ++++++++++++++++++ .travis.yml | 1 + CONTRIBUTING.md | 12 +- README.md | 1 + requirements.txt | 3 + setup.cfg | 2 + tabpy/models/deploy_models.py | 7 +- tabpy/tabpy_server/handlers/base_handler.py | 6 +- .../handlers/endpoints_handler.py | 1 - .../handlers/evaluation_plane_handler.py | 10 +- tabpy/tabpy_server/management/util.py | 1 - tabpy/tabpy_server/psws/callbacks.py | 1 - tabpy/tabpy_tools/client.py | 5 +- tabpy/tabpy_tools/rest.py | 6 +- tabpy/tabpy_tools/rest_client.py | 2 +- tests/integration/integ_test_base.py | 3 +- .../test_custom_evaluate_timeout.py | 4 +- .../test_deploy_and_evaluate_model.py | 2 - .../test_deploy_and_evaluate_model_ssl.py | 2 - .../test_deploy_model_ssl_off_auth_off.py | 2 - .../test_deploy_model_ssl_off_auth_on.py | 2 - .../test_deploy_model_ssl_on_auth_off.py | 2 - .../test_deploy_model_ssl_on_auth_on.py | 1 - .../server_tests/test_service_info_handler.py | 2 +- tests/unit/tools_tests/test_rest.py | 1 - tests/unit/tools_tests/test_schema.py | 2 - 26 files changed, 141 insertions(+), 48 deletions(-) create mode 100644 .github/workflows/test_coveralls_codestyle.yml create mode 100644 requirements.txt create mode 100644 setup.cfg diff --git a/.github/workflows/test_coveralls_codestyle.yml b/.github/workflows/test_coveralls_codestyle.yml new file mode 100644 index 00000000..7d6045c9 --- /dev/null +++ b/.github/workflows/test_coveralls_codestyle.yml @@ -0,0 +1,108 @@ +name: CI + +on: [push, pull_request] + +jobs: + ubuntu-build: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.6, 3.7] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Python Style Checker + uses: andymckay/pycodestyle-action@0.1.3 + + - name: Test with pytest + run: | + pip install pytest pytest-cov coveralls coverage==4.5.4 + pytest tests/unit --cov=tabpy --cov-append + pytest tests/integration --cov=tabpy --cov-append + coveralls + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} + + - name: Markdownlint + uses: nosborn/github-action-markdown-cli@v1.1.1 + with: + files: . + + windows-build: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.7] + os: [windows-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Test with pytest + run: | + pip install pytest pytest-cov coveralls + pytest tests/unit + pytest tests/integration + + mac-build: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.7] + os: [macos-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Test with pytest + run: | + pip install pytest pytest-cov coveralls + pytest tests/unit + pytest tests/integration + \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 13cbf053..0e09ab14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,6 +2,7 @@ os: linux language: python python: 3.6 install: + - python -m pip install --upgrade pip - pip install pytest pytest-cov coveralls - npm install -g markdownlint-cli script: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 63518658..6399188e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ and run it locally. These are prerequisites for an environment required for a contributor to be able to work on TabPy changes: -- Python 3.6.5: +- Python 3.6 or 3.7: - To see which version of Python you have installed, run `python --version`. - git - TabPy repo: @@ -57,16 +57,10 @@ be able to work on TabPy changes: cd TabPy ``` -4. Register TabPy repo as a pip package: - - ```sh - pip install -e . - ``` - -5. Install all dependencies: +4. Install all dependencies: ```sh - python setup.py install + pip install -r requirements.txt ``` ## Tests diff --git a/README.md b/README.md index a744f703..63200865 100755 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ [![Scrutinizer Code Quality](https://scrutinizer-ci.com/g/tableau/TabPy/badges/quality-score.png?b=master)](https://scrutinizer-ci.com/g/tableau/TabPy/?branch=master) [![Python 3.6](https://img.shields.io/badge/python-3.6-blue.svg)](https://www.python.org/downloads/release/python-360/) +[![Python 3.7](https://img.shields.io/badge/python-3.7-blue.svg)](https://www.python.org/downloads/release/python-370/) ![Release](https://img.shields.io/github/release/tableau/TabPy.svg) diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 00000000..6e1f77e0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,3 @@ +# installs dependencies from ./setup.py, and the package itself, +# in editable mode +-e . \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..1df793aa --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[pycodestyle] +max-line-length = 88 \ No newline at end of file diff --git a/tabpy/models/deploy_models.py b/tabpy/models/deploy_models.py index 148090f0..86b1310f 100644 --- a/tabpy/models/deploy_models.py +++ b/tabpy/models/deploy_models.py @@ -9,11 +9,8 @@ def install_dependencies(packages): pip_arg = ["install"] + packages + ["--no-cache-dir"] - if hasattr(pip, "main"): - pip.main(pip_arg) - else: - from pip._internal import main - pip._internal.main(pip_arg) + from pip._internal import main + pip._internal.main.main(pip_arg) def main(): diff --git a/tabpy/tabpy_server/handlers/base_handler.py b/tabpy/tabpy_server/handlers/base_handler.py index cc749227..dbbb6371 100644 --- a/tabpy/tabpy_server/handlers/base_handler.py +++ b/tabpy/tabpy_server/handlers/base_handler.py @@ -311,8 +311,7 @@ def _get_credentials(self, method) -> bool: # No known methods were found self.logger.log( logging.CRITICAL, - f'Unknown authentication method(s) "{method}" are configured ' - f'for API "{api_version}"', + f'Unknown authentication method(s) "{method}" are configured ', ) return False @@ -368,8 +367,7 @@ def _validate_credentials(self, method) -> bool: # No known methods were found self.logger.log( logging.CRITICAL, - f'Unknown authentication method(s) "{method}" are configured ' - f'for API "{api_version}"', + f'Unknown authentication method(s) "{method}" are configured ', ) return False diff --git a/tabpy/tabpy_server/handlers/endpoints_handler.py b/tabpy/tabpy_server/handlers/endpoints_handler.py index bd269d3a..66132dd2 100644 --- a/tabpy/tabpy_server/handlers/endpoints_handler.py +++ b/tabpy/tabpy_server/handlers/endpoints_handler.py @@ -11,7 +11,6 @@ from tabpy.tabpy_server.common.util import format_exception from tabpy.tabpy_server.handlers import ManagementHandler from tornado import gen -import tornado.web class EndpointsHandler(ManagementHandler): diff --git a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py index ae4a55f3..3b1d1ce6 100644 --- a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py @@ -127,10 +127,16 @@ def _call_subprocess(self, function_to_evaluate, arguments): # Exec does not run the function, so it does not block. exec(function_to_evaluate, globals()) + # 'noqa' comments below tell flake8 to ignore undefined _user_script + # name - the name is actually defined with user script being wrapped + # in _user_script function (constructed as a striong) and then executed + # with exec() call above. if arguments is None: - future = self.executor.submit(_user_script, restricted_tabpy) + future = self.executor.submit(_user_script, # noqa: F821 + restricted_tabpy) else: - future = self.executor.submit(_user_script, restricted_tabpy, **arguments) + future = self.executor.submit(_user_script, # noqa: F821 + restricted_tabpy, **arguments) ret = yield gen.with_timeout(timedelta(seconds=self.eval_timeout), future) raise gen.Return(ret) diff --git a/tabpy/tabpy_server/management/util.py b/tabpy/tabpy_server/management/util.py index 7590461b..cb9b7709 100644 --- a/tabpy/tabpy_server/management/util.py +++ b/tabpy/tabpy_server/management/util.py @@ -5,7 +5,6 @@ from ConfigParser import ConfigParser as _ConfigParser except ImportError: from configparser import ConfigParser as _ConfigParser -from datetime import datetime, timedelta, tzinfo from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters diff --git a/tabpy/tabpy_server/psws/callbacks.py b/tabpy/tabpy_server/psws/callbacks.py index 9ffff32c..4b1fe14e 100644 --- a/tabpy/tabpy_server/psws/callbacks.py +++ b/tabpy/tabpy_server/psws/callbacks.py @@ -1,5 +1,4 @@ import logging -import sys from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters from tabpy.tabpy_server.common.messages import ( LoadObject, diff --git a/tabpy/tabpy_tools/client.py b/tabpy/tabpy_tools/client.py index a5a99bbd..7ee069a2 100755 --- a/tabpy/tabpy_tools/client.py +++ b/tabpy/tabpy_tools/client.py @@ -1,10 +1,11 @@ +import copy from re import compile import time import requests from .rest import RequestsNetworkWrapper, ServiceClient -from .rest_client import RESTServiceClient, Endpoint, AliasEndpoint +from .rest_client import RESTServiceClient, Endpoint from .custom_query_object import CustomQueryObject import os @@ -313,7 +314,7 @@ def _gen_endpoint(self, name, obj, description, version=1, schema=[]): "methods": endpoint_object.get_methods(), "required_files": [], "required_packages": [], - "schema": schema, + "schema": copy.copy(schema), } def _upload_endpoint(self, obj): diff --git a/tabpy/tabpy_tools/rest.py b/tabpy/tabpy_tools/rest.py index 8959fc11..f200c250 100755 --- a/tabpy/tabpy_tools/rest.py +++ b/tabpy/tabpy_tools/rest.py @@ -101,7 +101,7 @@ def POST(self, url, data, timeout=None): response = self.session.post( url, data=data, - headers={"content-type": "application/json",}, + headers={"content-type": "application/json"}, timeout=timeout, auth=self.auth, ) @@ -121,7 +121,7 @@ def PUT(self, url, data, timeout=None): response = self.session.put( url, data=data, - headers={"content-type": "application/json",}, + headers={"content-type": "application/json"}, timeout=timeout, auth=self.auth, ) @@ -418,6 +418,6 @@ def __new__(cls, value): return super(enum, cls).__new__(cls, value) - enum = type("Enum", (enum_type,), {"values": values, "__new__": __new__,}) + enum = type("Enum", (enum_type,), {"values": values, "__new__": __new__}) return enum diff --git a/tabpy/tabpy_tools/rest_client.py b/tabpy/tabpy_tools/rest_client.py index 379a3720..eb0ef211 100755 --- a/tabpy/tabpy_tools/rest_client.py +++ b/tabpy/tabpy_tools/rest_client.py @@ -50,7 +50,7 @@ class Endpoint(RESTObject): def __new__(cls, **kwargs): """Dispatch to the appropriate class.""" - cls = {"alias": AliasEndpoint, "model": ModelEndpoint,}[kwargs["type"]] + cls = {"alias": AliasEndpoint, "model": ModelEndpoint}[kwargs["type"]] """return object.__new__(cls, **kwargs)""" """ modified for Python 3""" diff --git a/tests/integration/integ_test_base.py b/tests/integration/integ_test_base.py index 3e7104c2..9453a1f6 100755 --- a/tests/integration/integ_test_base.py +++ b/tests/integration/integ_test_base.py @@ -1,7 +1,6 @@ import coverage import http.client import os -from pathlib import Path import platform import shutil import signal @@ -292,7 +291,7 @@ def deploy_models(self, username: str, password: str): input_string = f"{username}\n{password}\n" outfile.write(f"--<< Input = {input_string} >>--") coverage.process_startup() - p = subprocess.run( + subprocess.run( [self.py, path, self._get_config_file_name()], input=input_string.encode("utf-8"), stdout=outfile, diff --git a/tests/integration/test_custom_evaluate_timeout.py b/tests/integration/test_custom_evaluate_timeout.py index 04bbb655..3a38e21d 100644 --- a/tests/integration/test_custom_evaluate_timeout.py +++ b/tests/integration/test_custom_evaluate_timeout.py @@ -15,8 +15,8 @@ def test_custom_evaluate_timeout_with_script(self): """ headers = { "Content-Type": "application/json", - "TabPy-Client": "Integration test for testing custom evaluate timeouts with " - "scripts.", + "TabPy-Client": "Integration test for testing custom evaluate timeouts " + "with scripts.", } conn = self._get_connection() diff --git a/tests/integration/test_deploy_and_evaluate_model.py b/tests/integration/test_deploy_and_evaluate_model.py index 90e73c6c..91b071a5 100644 --- a/tests/integration/test_deploy_and_evaluate_model.py +++ b/tests/integration/test_deploy_and_evaluate_model.py @@ -1,6 +1,4 @@ import integ_test_base -import subprocess -from pathlib import Path class TestDeployAndEvaluateModel(integ_test_base.IntegTestBase): diff --git a/tests/integration/test_deploy_and_evaluate_model_ssl.py b/tests/integration/test_deploy_and_evaluate_model_ssl.py index dd5c7d8f..2de1c350 100755 --- a/tests/integration/test_deploy_and_evaluate_model_ssl.py +++ b/tests/integration/test_deploy_and_evaluate_model_ssl.py @@ -1,7 +1,5 @@ import integ_test_base import requests -import subprocess -from pathlib import Path class TestDeployAndEvaluateModelSSL(integ_test_base.IntegTestBase): diff --git a/tests/integration/test_deploy_model_ssl_off_auth_off.py b/tests/integration/test_deploy_model_ssl_off_auth_off.py index e532ae49..b754ccfc 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_off.py @@ -1,6 +1,4 @@ import integ_test_base -import subprocess -from pathlib import Path class TestDeployModelSSLOffAuthOff(integ_test_base.IntegTestBase): diff --git a/tests/integration/test_deploy_model_ssl_off_auth_on.py b/tests/integration/test_deploy_model_ssl_off_auth_on.py index 00040934..09e5c122 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_on.py @@ -1,7 +1,5 @@ import integ_test_base import base64 -import subprocess -from pathlib import Path class TestDeployModelSSLOffAuthOn(integ_test_base.IntegTestBase): diff --git a/tests/integration/test_deploy_model_ssl_on_auth_off.py b/tests/integration/test_deploy_model_ssl_on_auth_off.py index 87596f20..f7a81709 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_off.py @@ -1,7 +1,5 @@ import integ_test_base import requests -import subprocess -from pathlib import Path class TestDeployModelSSLOnAuthOff(integ_test_base.IntegTestBase): diff --git a/tests/integration/test_deploy_model_ssl_on_auth_on.py b/tests/integration/test_deploy_model_ssl_on_auth_on.py index 19e17730..6e833162 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_on.py @@ -1,7 +1,6 @@ import integ_test_base import base64 import requests -import subprocess class TestDeployModelSSLOnAuthOn(integ_test_base.IntegTestBase): diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index 585e47b4..ece2eea5 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -120,7 +120,7 @@ def test_given_tabpy_server_with_auth_expect_correct_info_response(self): self.assertTrue("features" in v1) features = v1["features"] self.assertDictEqual( - {"authentication": {"methods": {"basic-auth": {}}, "required": True,}}, + {"authentication": {"methods": {"basic-auth": {}}, "required": True}}, features, ) diff --git a/tests/unit/tools_tests/test_rest.py b/tests/unit/tools_tests/test_rest.py index 9543005c..59f74234 100644 --- a/tests/unit/tools_tests/test_rest.py +++ b/tests/unit/tools_tests/test_rest.py @@ -1,7 +1,6 @@ import json import requests from requests.auth import HTTPBasicAuth -import sys from tabpy.tabpy_tools.rest import RequestsNetworkWrapper, ServiceClient import unittest from unittest.mock import Mock diff --git a/tests/unit/tools_tests/test_schema.py b/tests/unit/tools_tests/test_schema.py index 4101b79a..ba131696 100755 --- a/tests/unit/tools_tests/test_schema.py +++ b/tests/unit/tools_tests/test_schema.py @@ -1,6 +1,4 @@ import unittest -import json -from unittest.mock import Mock from tabpy.tabpy_tools.schema import generate_schema From fbf8f312bdfe971564efc6026b40d1f0de8d2e2e Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Tue, 24 Dec 2019 14:18:34 -0800 Subject: [PATCH 12/61] Documentation improvements (#385) * Delete .travis.yml * Fix Ctrl+C failing on extra parameter in signal handler * Remove outdated documentation for how to configure connection * tabpy.py to use docopt * Update tabpy-user with docopt * Update CHANGELOG * Fix code style --- .../workflows/test_coveralls_codestyle.yml | 2 +- .pep8speaks.yml | 2 +- .travis.yml | 14 --- CHANGELOG | 12 +++ CONTRIBUTING.md | 18 +--- docs/TableauConfiguration.md | 40 +++------ docs/img/external-service-configuration.png | Bin 11340 -> 0 bytes docs/security.md | 6 +- docs/server-config.md | 12 +-- setup.cfg | 8 +- setup.py | 3 +- tabpy/VERSION | 2 +- tabpy/tabpy.py | 18 +++- tabpy/tabpy_server/app/app.py | 22 +---- tabpy/tabpy_tools/client.py | 1 - .../{user_management.py => tabpy_user.py} | 80 ++++++------------ tests/unit/server_tests/test_config.py | 14 --- .../server_tests/test_endpoint_handler.py | 9 -- .../server_tests/test_endpoints_handler.py | 9 -- .../test_evaluation_plane_handler.py | 8 -- .../server_tests/test_service_info_handler.py | 14 --- 21 files changed, 95 insertions(+), 199 deletions(-) delete mode 100644 .travis.yml delete mode 100755 docs/img/external-service-configuration.png rename tabpy/utils/{user_management.py => tabpy_user.py} (62%) diff --git a/.github/workflows/test_coveralls_codestyle.yml b/.github/workflows/test_coveralls_codestyle.yml index 7d6045c9..4f91b6ea 100644 --- a/.github/workflows/test_coveralls_codestyle.yml +++ b/.github/workflows/test_coveralls_codestyle.yml @@ -1,6 +1,6 @@ name: CI -on: [push, pull_request] +on: [pull_request] jobs: ubuntu-build: diff --git a/.pep8speaks.yml b/.pep8speaks.yml index 8daa7cd4..acf60421 100644 --- a/.pep8speaks.yml +++ b/.pep8speaks.yml @@ -1,4 +1,4 @@ # File : .pep8speaks.yml flake8: - max-line-length: 88 + max-line-length: 98 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 0e09ab14..00000000 --- a/.travis.yml +++ /dev/null @@ -1,14 +0,0 @@ -os: linux -language: python -python: 3.6 -install: - - python -m pip install --upgrade pip - - pip install pytest pytest-cov coveralls - - npm install -g markdownlint-cli -script: - - pip install -e . - - py.test tests/unit --cov=tabpy --cov-append - - py.test tests/integration --cov=tabpy --cov-append - - markdownlint . -after_success: - - coveralls diff --git a/CHANGELOG b/CHANGELOG index dcd61df3..59b44e46 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,6 +4,18 @@ ### Improvements +- Minor code improvements. +- Documentation updates with referencing Tableau Help pages. +- + +### Bug Fixes + +- Fixed failing Ctrl+C handler. + +## v0.8.9 + +### Improvements + - Added Ctrl+C handler - Added configurable buffer size for HTTP requests - Added anvoa to supported pre-deployed models in tabpy diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6399188e..ba332aea 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -40,7 +40,7 @@ be able to work on TabPy changes: - Create a new branch for your changes. - When changes are ready push them on github and create merge request. - PIP packages - install all with - `pip install pytest pycodestyle autopep8 twine coverage --upgrade` command + `pip install pytest flake8 twine coverage --upgrade` command - Node.js for npm packages - install from . - NPM packages - install all with `npm install markdown-toc markdownlint` command. @@ -60,6 +60,7 @@ be able to work on TabPy changes: 4. Install all dependencies: ```sh + python -m pip install --upgrade pip pip install -r requirements.txt ``` @@ -145,21 +146,10 @@ Access-Control-Allow-Methods = GET, OPTIONS, POST ## Code styling -`pycodestyle` is used to check Python code against our style conventions: +`flake8` is used to check Python code against our style conventions: ```sh -pycodestyle . -``` - -For reported errors and warnings either fix them manually or auto-format files with -`autopep8`. - -Run the tool for a file. In the example below `-i` -option tells `autopep8` to update the file. Without the option it -outputs formatted code to the console. - -```sh -autopep8 -i tabpy-server/server_tests/test_pwd_file.py +flake8 . ``` ## Publishing TabPy Package diff --git a/docs/TableauConfiguration.md b/docs/TableauConfiguration.md index 69f02100..2e3fe0cc 100755 --- a/docs/TableauConfiguration.md +++ b/docs/TableauConfiguration.md @@ -20,40 +20,24 @@ configure Tableau to use this service for evaluating Python code. ### Tableau Desktop -In Tableau Desktop version 10.1 or later: - -1. Go to Help->Settings and Performance->Manage External Service Connection... -2. Enter the Server (localhost if running TabPy on the same computer) and the - Port (default is 9004). - -![Screenshot of Configuration on Tableau Desktop](img/external-service-configuration.png) +To configure Tableau Desktop version 10.1 or later to connect to TabPy server +follow steps at Tableau +[Configure an external service connection](https://help.tableau.com/current/pro/desktop/en-us/r_connection_manage.htm#configure-an-external-service-connection) +documentation page. ### Tableau Server 2018.2 and Newer Versions To configure Tableau Server 2018.2 and newer versions to connect to TabPy server -use [TSM command line tool](https://onlinehelp.tableau.com/current/server/en-us/tsm.htm). - -To configure a non secure connection to TabPy server configuration: - -```sh -tsm configuration set -k vizqlserver.extsvc.host -v -tsm configuration set -k vizqlserver.extsvc.port -v -tsm pending-changes apply -``` - -To configure a secure connection to TabPy server use `tsm security vizql-extsvc enable` -command as described at -[TSM Security documentation page](https://onlinehelp.tableau.com/current/server/en-us/cli_security_tsm.htm#tsm_security_vizql-extsvc-ssl-enable). - - - -```sh -tsm security vizql-extsvc-ssl enable --connection-type --extsvc-host --extsvc-port [options] [global options] -``` +follow instructions on Tableau +[Configure Connections to External Services](https://onlinehelp.tableau.com/current/server/en-us/tsm.htm) +page. - +Specific details about how to configure a secure connection to TabPy, enable or +disable connections and other setting can be found at Tableau +[TSM Security documentation](https://onlinehelp.tableau.com/current/server/en-us/cli_security_tsm.htm#tsm_security_vizql-extsvc-ssl-enable) +page. -For how to configure a secure TabPy instance follow instructions at +For how to configure TabPy instance follow instructions at [TabPy Server Config documentation](server-config.md). ### Tableau Server 2018.1 and Older Versions diff --git a/docs/img/external-service-configuration.png b/docs/img/external-service-configuration.png deleted file mode 100755 index d997327200e422d6444e53492de03fccbe20a2f0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 11340 zcma)iWmH_t)@|?(?gXcS-~o6pozNc` zM`bC|SLI_Qd(a1XbCFLXuU=I}p*$KQK%ZaRNozU2dWGKo=Yr|CEi{3?1UgA*I*B=$ z8ai3n+EA!kSew3LXJzB!V`JrG_ZTUYyg-CA@~5#^o4+$a20a~CktsZgJ3cKv%!p$q)hsli zmyO;CA2z$=w2NRK7Fht03cElI1Mk~pp41IE>0%H!5ZV}zyK$E9a}z$yf_|f2r?1tR z3OoQvEwC}uyG~5ZVfLMCrwR*^3YkWal9~x|W36|AA7SnB?&9`xTH$QLJj=on+5f_! zI%^jVQ*=POi|47zxc_S!tD*l2h?rX*#qThU$A}WpRT=6hL#aarPYJtmD**^pDUe^k z+?)PlWi`_eou=H)pJ_ItK0a;NRh#`$^VhvBK1#L{xZ?2XRPcray&d%MX4=wn;%jjV zLX%YfAQ1ZFbwm5?S=+<%$;Nh??c75kCj0M0a0AA0B6CgD6q3sE>eI2S+x=dF^Y4cC zC+m?+L1G+Q`5rtC-JAwW06QP+wA5fKL%0s=$Ypd25jGIh^J-S(?679s=xDJ{v+}ba z2s&G063r2Q>ua3>ZiiLi7Ev;WyulJ-rGn$&i^l2< z!=XLxLSb1u{&n&b6S^$0p%{1u_aVZMFy7wY%{9y%tT_fFY|lJO8`y9GxE)RUr_T>3 zo10O>+qIa$Kx&xjqT{{2Nm3iE07alQzI_zxNMXl!RB_bnlhvqS$IZha2>@_!bUe$8 z=*9SssD02r|FrXSZ#W*0*KOqs3ZFb1E0+}&oTDKXvN8{wUmh%UJ=X6M18!$m`k!d} zH*S}^kRo(!nj~y9rp5Zw+o1i(q+$PRB8MyqMMhXqFN)Ln>SvO6D>MiF4KyyBJeZm+ z)D3(%D%i0uEajUUUVZ&II3q9@+R08D=mF$Dw>hi9 z$^iIH^GE=7TB8)A=ph1M&^E-+U~{^Dlk?>hdP8eHm6!V^ugbP5lHJr>`9l|Mq(n4P z9~WNNmTi1m&ci_S6oyR)5}rMu6^p=p8&84w@=-S}kUA9|{v&jFHAvGj1S1%tNv^Q9hG z$;kA+J;e8wCXpSV!$nUQq@J9<@Jts*zaLghgunVcIyewx^8HQK^!BZCZory>WkD!zGl>iWFd)JXCI^&BFL#?FqT6^&!#Bh z=N~!Rj@suWR1=u`R$OoLNQRyC^h%M*%640P8oPa?lK9BGyb)Y?uiztX1?TcqTuJo7 z>)owz-ui*muk9a+xdpelYd#<1XI>W~G11I)@ALaUx12yt#`V?J)t!6>AB5lKFu9_v zY_!4S>Z`Yq0IfGtVyDJ2?|Kxi&M=?%&{;`A`%=Tp@)|3o=hA)$#B_!ZtY4mY-wPb4 zx;yrrmpAWUUfJ_mYU(a{PZKTG)L)REs&Lnt{l0vzN$&^!4Sm73${1U8$ZDzM(|N2D z=pdQzw4Z^WK7jf^BiM3SY^Y+HSjoQ26vIQNizH zGGQy{35`;5R*rXhEq~1mCz0b7CwWPAdbjMORj$w zOlYlU2T%bo7W^#3IW&k^ABG#7vtc1Skc1 zz^0b8K6c!c&U312{p6(@x{T&KHW=%6`&kP-ClV9?nfuw4_c}rSN~b+pyAm7n@KFg* zEd%h*opdDtOq^&4@qMUfhQ)=&7Mw0n=;uFaF(&_kq^wVWW5-@KSb=c6`Kd%Ih)HgMH$f&l9po7rR}^7c)xM;S$u&P6F|0yxV0m<5FszN5 zcuH>V%0$8c=UaKHHCF2utw5(Ra^tNBsgs|M#zgj_ywn|EKt1iAVD(HNUKQ7M-eFY+N3f;6yyi zTN*&fr>ue6!}eQVyVT7*R#!Z_+d@LEiGY*Td-vc#%sK|0It36EYQjU$h^Q76NmrBM zWMBYy{;Tb!xeNQOy1E8eA;rFk|<2D`E}f?mQ&2bYILe_ z1RqHqj`c>maHU;XxAz5O8cuaRh#d>Mms$H6^$p8>)NA|qdv7y=yXmIeE4n8l{TMUWYL*x-_cvuu<)#3j z*UGW!>me0ivM0jIF|i>sE?;v!#M!jw`H#*=*O7FpXR%z!Z>b}twbHa8-ViY;8oa}(;kkaNdY}|o4Pt+<9*%<_qK^Ll_xjLR+v*D}zrDiF7I)6wXh-I> z$|7yZPF2PHzKFL3=Uni2B8R#9+lzh6f&M-zv7rKzPmn$@XBkF$F&~2F?nH$g5=D)o z!1-M1NbTZT?=(tHQ#!S`k?gJVTOGdgDKoQNjGumcvO1R>)M2we@aYAz^8m6MbhN0) zBL}#%qP)n|4=E5!9f4NA09&g{n-WbH3L;mD0d>xP^%L0lIXJN}`K51gXBRPCiu(6w zQ$pPI3Gq|S>k4u4PB(6c*_VwYLXt08!3j9W?qrdLoy}k7VX50aq8xx5AqPrI`>_)X z(@YgeE&TzcM)KuE)6!a{`d^gF9JoCAhJJWq8}L$?--UlnpsJb8p52s)O#cq_#vF6x zxPS$5EeR$!Cp3+zhAYiS0%U+)lCtpEl;D7&Q& z{47)t6Af6E36T(7Y7RN!PRcz0hP#IqR#1nRd5?TK4Qu~#tPXWPvy{kw_aLb&k2G13 z6U+K8d@iSt@uqla&6%e$e<_Qy*b~tG!Bm?1#12(5Gcelc&+z#RLYy$zF*uIIgxMo0 z1-o=qZFSKe9!ANwn8XQnyQmav%Mn-39o#JEn)r33Flu-1`koz^hTPTmc+Ihb&}T+e z1XTm+vlw|bHMfHbhTv=$L@jf4B#D&@qWt}iJ0rggW6G}ZjY|sHx?XhmMb)(!E(Xb$ z6fAeGaZLfK#r`1bB2i}=)@X{Ow5A;igONK-oC%}D`G5I-%-TLQ4iBrtVF?t zJ>TtQSwgvkfY8&WN;T$dKg_){`1{i2hse)KtZqY`>_hx`R z$9h<-|FeDobyCh3fapRdH+HY9x@(uhCq zr7ST@QM(yy&u-A>S3W#ZX|ou8qwL`GXW=)*)LS}okFs0ujRA`YKq^wu|7xhFj!saO zqj6|oK&}dpq~6}-i$D`Ovm1hkZRUZ|82ow(gLsu2CUsb@)?2Q2 zh4Iy71v1=({57kz=Z1w||JoC`RT(UCjyV(4J=+))(=sKgV9jp0_Z3Pzza`m0@pu6Y zJ0<83MA56gu8J|ZU3bV6h}i=4AVGD|W(+X_9Fo`lK}Z{1hY>i8yW({qd5Je%#Jr}B zux_JB;Acj4<2G+O_7L1mnjxw37w)hKMN1zIzArtBsDhs>J+=5t4)s$(n}=m}xkDsU zaVzBGLRo+*9P-MLL^(HKdbZ6c973)-CY{vn3iIyJ*Y zHucD$BcuxvEb7a2g`h z!1!`381mTP2Ir_)$4o&=1nFs&9le`%Gb-V^Wt~@qt*ZVCY7WHn`3&L-(_OkyW!@gI z=w-um+0E4D-y9Ly?~1frpPZ<1B;LQ`Z~ozJE$$OA+s5~`g^v4(yU<&!!Z^b`>Yesu z1$Sf!x7|#|UQM*{s=HO3r{MON7(gf1JJCvP1yhqW3Vi4(cpgi5R4FT{<_3;Lavw z6@#gQq5o?yrU=P0qdS5iK|Om#JwQE*fYX6J`0_r8-kVu^msijS*Osnd=jbp@@Qfri z%Ci!~&rW!Z?0JKB#*u8d&Vt0W#Qec}urA50Y#+fOdF3t>-C>?cpc+Vic&7-l0Ly=QWU#ylp@eR3O43bqci~p0>pcb7|UuGIPssZ)+1}v54Q-}pj zuEQ7xEeO6C7bDqbg=0YIsL5BA1qw)m9*his8o^T>WBJvPqV4F+Ih=F?pLoI6Wki3w&nd$vSrxo{Y(+|zy4~71HXSmQNM zOLp`Qv=dd+2@%&CGh$1KFPGA`S~6~RhywoRn(@Yi)n*yV2K;AWqzT&?zSq=L`t z+>EO@zh}L5yshS;5UeVIT{p4>EeCvZEDh4*)!)m&fecWLaK=k_pgz=?8Ln0rWETXe zDLBMA03y`fXx>p^yz4wFu`?DT>S|R4>(9s14+pk!Ew}WY5t3hw-z5pWGhjxxz`^Qz zy6Z#mwb!quBC`3sWwrQIsH)y~NwMZC7*20go%Ax>ZH~Qafl7es`f@xrYGC^3S zzF~%zRG6xqka{2pCFykU2mG@~LOv6Y|80HM#nb3-Ug#Knb!HubLp`5#6bjr)wUwcLIi)Jh=e`h3v7=oyF^ ztAEN8O;vf1BpkX3rJOwqLkc)`Ep7v|+6G^{Wkd8k$roR&`bi82EHRO;n5_u=qze`5 zU$cki_ojAUT2`P{rxO}U=Z zEpINd7gMax>!1!`%J-|+4x zZGv}P)e*W%w4D)Qux~!cm*vaNedyPVw)a1Wye49gk(qt{u{C z4!dow!u>h)YaHcw=8JY$`|9RTU5D&@)Xk{CtR~?ldE+tg-s2Ss(fAP=Kn5FYTT?lJ zt$2q8WFYx!?Vof1|HuzV>a^0X&8oegvsf&~?gu^|h@F$OUhQ657P;mwml0!%Y8 zfh476B_eqYVj16Y^93-_bV2EL%)2_4=`dRimpLe)Z+OMS{ZrL>() z&#@J%qJgO&vPZ2mw2CA|m{Y6S0ns1S>H40I2Xyhty?p0I|>@eeon3bZDScUGH!=|&@rfK)CrYPHtG(Sjm39gU!tgp-;an5 zCDv4;UT}F#6mK9b%i`WtIzT>pM&%Qc`y}gEw8`CBP!bXjgnlk1LJI|i4%Wsj3|&rs z!=2sd3<&qQ>R1rjJtnFKHk>Q$?yJjpr-h{Oq9|>vK@{i5Sx5&|(7w5VK+Z8BBG*p4 zAAmX!<{!9ZJE}nrLZp04$qsgWKDgA%nSSMI6Rmv%{Ojcqb@SsZo-bMMfDOAZMHYLK zLlJ)Z1d22&Vv?_VOj4yI{E%HG(nEaDHFG?*_B(4aoLMScvBnR|A#0R^!d75JC|91H z9`z&f5GXb{WceO&{m#+@s7)dW&fFnhMi`4J5C#=zF$iM48!Lt5ylDCQol-@c7?|<9 zG3=*6&TdgHT{-a|Zr!m%tfr+cSeRrfr}1Dj?@&^cq}MfL*fTD$<>UN}_k$q>bE64K zbb}hY{9U*9JdQsJPTF5!?&B9T1zpi!Zg;7?F3m|g%;h%cl0k|E%wTvav4vN)?Gil; z4~I@xb=2+mzqas=XY(kBMh3Y&;pSGAqFXwuW)$jIHY}Cnu+?A^p-sm3CqRebxYC(r z`}=l?K25-srb=XvzDp0fwKnBAFG>C#$(RR3vrP@GDYl)QKhFVvB#13CtIBtR_NgZ*A)X*2%=*c5nHW)ovj66 zXq!o*usg6CVjN4R!7t{8)ziGT6xEohGX10%%R@b$efjCjqV>jg?r6L1569MW;U#-H zXy1DVaN!^(Y3OEoR^5nJit35W5jPV-KlNH0C(*%`U%5c<7bxxTDLgKyySsSI2YywpRo*XBLDp?;1>NUl)p=yUf{y);4WP(&<$JtJJ92O0{$Emxf?A*WU4a5r9VY{qGh& zzEQPpYhc4%ZLcIjgZ9(-TI8^Ln(K=1XB1`Znlm&`30T9J{Q>is!zVh0@~lV-U9H-` z*L7Na@0Ed#n6yV{5*X?iA+%k`MqdV0wB|#7Cg~A7m_PE=e+T1wL)a9IPhC8;x&iZ# zH`JuhlZ3!MgqPj#MyzzFA^QLsmRm7rC5C|@(T(tQmWwPFEx0S zbW8XkAH%Udl7*Wn9*@a-@XGwU3z6fB{QMj)w%NuJU+6S8&F<-kCIHZ+k&w-30_*}< zpRpl9q)!^R9$(9OzZLQ0R)lYXkq5Xg6YY<_=s&d|+Tc$sRM}(A{yh=gKe8 zNMAAk4x?+C-Sq%*>%JzhgZ=qw?&NXZ4zm(Em|5<5m>@^-xSgBW{i0@)@$2XF=p!fq zvuFvFyBwdaZTK_>z(v4Z5THjOw8mZAWe*OFtYekuSL(#}J*9Kmw|rb&CX zEIpJUz&D*Yb_qj;H-eD0gg3eg-hyT~#oj;A7u;&|xBmZfci}hC{5S^R)uyb{M!DgE9}bQ$^haI01)ujcoj^-ZS|gHy$PwnU8DlEcbUi^?2AlUvKX; z2Zn`8>q45$!Qqr4`Uz3ag@Iy`Tpq+_kFJBM-PjFS0pYXhvq^D;cA|AEne!eB+b-iHj0+7LnM$!9cid2-8 zlng^m(5*C{!s-Mo(I4UlIO?pyTviL!y|cdR>gq;=v2UTGH+y;G2C}Zt9VD5@KDXX} zv(NDGchws~52yCps^}@BN)?E6O7j9_IEjj%AB_HXFt?$B!1Ly~w?E6vzpl8-6xmAS zxMzw?Gvt{1Tuup<^0WgEWA{BYRz5Wk95n3;d<~V4rz8JFyBox$g#NuUTEmQ#A%u)hn(X!jB;6n?q`Tke{FZD=h{{IHl)t+^;RbzoPR#jEDigp>ttf@jjpDtb>sIMR{2 zt^b5qzH(i5+s9p>t9PBiS&@Q;&F!xjiy_BpmgKn1Pq1DL5jw28zyohyWY4}mj!v#A z<(SKEaz`@@>UnB#+zu1r;)3Mcw#;AkrHMcX>{<0x z|9pp9|7YB1zbgeCvAjM|CYI;ly;mBzJLZcwF+VsFG&fJmE(ar@lrvU!$0y7b85(l$ zKhM)V?kS|PDg;4aAWd!MjMY67{k|V^S=z?9Sg$|2wx8fU?zWXn9%?ui!pV$$@WBDL z7zp`Q-+Q%pdOzERkt@$3kxjVo=U6kDPP*ST$n}*8A@=(O_^#$J&a_u_YCIV>7(OuS z_!%@5j!(Ud@^ZQGiS%`cyqfcxP4U@fQeQd`U8WRv6dA8@SEsDMm@ppzonUZ`>#&~y zKkt2na?VG=?u8oj<)gfDyR!}RTWNA`!)5MOaZ^|B=3afxK=XN_@OcAYTuW}qh7AAX z;1c&)--1=?ZNn%X)6#bW)!&(T3w{Cyt5N1J@exG}*Y?qaoLs;g8O90%u695EIZiEz z;@JJD&Y`7n6YR^`%S>H&Pzd9}!u;887r$NSn89KHDuDh#*tfH7=gsici9S=@{irx) zeTfP9pm-GTUDx9TSArm3ldtlQ6Tc6>W&0fyerqKXqb_)5LX+tTXyN?kEQLSf5YO{* zT9;CzQN}MpF~dx5y^}s)>hYe z8cd(PW)j?wE9+g|33WRSryW?UP*9l(&s8V4h%)`z5F%_lI|L@hwDd27cj(z$&j(eI zkYAJ2EGrXQrHv4*nmsP!?G&7Yg-C~s(xofZiP?ZmyQ8HeLgbs4peJ4I8YffuPdG)S zvpm`$vR9s$m zD%I{m*4)ItRBa?mjo|O4c$Sb2N2DrE841%e-!K-i%%ioo@p9=%kv01myRSey_!iRp zfnolWcU=m`hwF-m@UMy(YlBIm8-;zfP0!4AzBtb(F*|2@{&~CZvO%)VpiMV0p$7O< z)ile&r5sT?RZx3z zzR%jlL3^cK2|y{Y>2l)dzIto6k!IXrPPcS{pDGnAcP=M2<(Vq2(R{~h+ZFZVNChg- zj9jR();Ysn$$ZAU^{Jmg*9qO0;Jkfy!t=eqQEIYcQ7<{gQ)nfPKe;>+j(k~iQ_bRV zFS}k2PZKS#IyJ93y^XR(rVn9vBzY&GtHtuE|Hjh4`(=EXA)}^rylp4-@4+MvHqQ2c zPO(vmNUfT9)-@m3M_b+07kGNw17^j1g;Nj)X%&wK*AM4Z{xPa#C=Vkt;H)bS-&SiL zAru5<4MT7fm;-m)op@~_bbcx!{yC>P>v47SpP%Q+D0 zOHReX(DPzF>?UOcV^+d1?SAS%%=ky`Qo;$ud3LM@k8CY`W*ZBO#2-}j%!urlcTji1 zZj^Jrk%5(&E$&~(1#{wc2cJ`elh03OkI@h3vGW57NlBYSj1{$FGS#=I-58S-6WInj zbdIqpu;2v#s#>JE9R4r?mF`xv=<|~gKF}nu+9|U|6wP2*iw{$c$88Tb{M3%0f9)%j z9{D-{&Jxe0|BE*Gq=iBm?&<%RyY&|*pjJY~z8mKLBj3q(8GXQ_2#oHO@+zBNr2*g2 zT!O{V;ndE-;^yO-61KK1K4%dIjn7;$6Eib`*Vir&(6&P3+%LBGR>_fjAGs!>vIm>R zRDSj0ttadM zllv!1d2v3>vYBF@Ir%^dUt=~pmFLgmwT|Mk&U2|FZniHC55!Uml|>y4w|Zu@#a#(!cL2=!IjC@tX)9v0_oeL3X~ey(STuBCHfvtb6SqNXS3OJa`DiAW-y zC#Wp=oDb4Idt)JEVX_qzLcN`z#w1Nvy@L^9-AQ}E8cxyAU2+0g@Nr!Fv6P5?ox~!T z1-9w!p#f~XQcyQI`kANqgU5Z}kQB z!uQ0|!;4P!d|*Xi0r)#wUXQwq`6tl^Mq~L+rx4SgINh7AT0vSkMk^!y)<(JxiD59~ zDkqqs -p -f +tabpy-user add -u -p -f ``` If the (recommended) `-p` argument is not provided a password for the user name @@ -229,11 +229,11 @@ will be generated and displayed in the command line. ### Updating an Account -To update the password for an account run `tabpy-user-management update` +To update the password for an account run `tabpy-user update` command: ```sh -tabpy-user-management update -u -p -f +tabpy-user update -u -p -f ``` If the (recommended) `-p` agrument is not provided a password for the user name diff --git a/setup.cfg b/setup.cfg index 1df793aa..beedadaf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,8 @@ +[pep8] +max-line-length = 98 + [pycodestyle] -max-line-length = 88 \ No newline at end of file +max-line-length = 98 + +[flake8] +max-line-length = 98 \ No newline at end of file diff --git a/setup.py b/setup.py index dc1265c9..4afa3212 100755 --- a/setup.py +++ b/setup.py @@ -67,6 +67,7 @@ def read(fname): "cloudpickle", "configparser", "decorator", + "docopt", "future", "genson", "jsonschema", @@ -82,7 +83,7 @@ def read(fname): "console_scripts": [ "tabpy=tabpy.tabpy:main", "tabpy-deploy-models=tabpy.models.deploy_models:main", - "tabpy-user-management=tabpy.utils.user_management:main", + "tabpy-user=tabpy.utils.tabpy_user:main", ], }, setup_requires=["pytest-runner"], diff --git a/tabpy/VERSION b/tabpy/VERSION index 021abec7..e6663d4c 100755 --- a/tabpy/VERSION +++ b/tabpy/VERSION @@ -1 +1 @@ -0.8.9 \ No newline at end of file +0.8.10 \ No newline at end of file diff --git a/tabpy/tabpy.py b/tabpy/tabpy.py index 3ee0df84..50acf799 100755 --- a/tabpy/tabpy.py +++ b/tabpy/tabpy.py @@ -1,9 +1,16 @@ """ -TabPy application. -This file main() function is an entry point for -'tabpy' command. +TabPy Server. + +Usage: + tabpy [-h] | [--help] + tabpy [--config ] + +Options: + -h --help Show this screen. + --config Path to a config file. """ +import docopt import os from pathlib import Path @@ -28,9 +35,12 @@ def read_version(): def main(): + args = docopt.docopt(__doc__) + config = args["--config"] or None + from tabpy.tabpy_server.app.app import TabPyApp - app = TabPyApp() + app = TabPyApp(config) app.run() diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 2e155a99..630f0f50 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -1,4 +1,3 @@ -from argparse import ArgumentParser import concurrent.futures import configparser import logging @@ -44,13 +43,9 @@ class TabPyApp: def __init__(self, config_file=None): if config_file is None: - cli_args = self._parse_cli_arguments() - if cli_args.config is not None: - config_file = cli_args.config - else: - config_file = os.path.join( - os.path.dirname(__file__), os.path.pardir, "common", "default.conf" - ) + config_file = os.path.join( + os.path.dirname(__file__), os.path.pardir, "common", "default.conf" + ) if os.path.isfile(config_file): try: @@ -98,7 +93,7 @@ def _create_tornado_web_app(self): class TabPyTornadoApp(tornado.web.Application): is_closing = False - def signal_handler(self, signal): + def signal_handler(self, signal, _): logger.critical(f"Exiting on signal {signal}...") self.is_closing = True @@ -167,15 +162,6 @@ def try_exit(self): return application - def _parse_cli_arguments(self): - """ - Parse command line arguments. Expected arguments: - * --config: string - """ - parser = ArgumentParser(description="Run TabPy Server.") - parser.add_argument("--config", help="Path to a config file.") - return parser.parse_args() - def _parse_config(self, config_file): """Provide consistent mechanism for pulling in configuration. diff --git a/tabpy/tabpy_tools/client.py b/tabpy/tabpy_tools/client.py index 7ee069a2..0bf43522 100755 --- a/tabpy/tabpy_tools/client.py +++ b/tabpy/tabpy_tools/client.py @@ -303,7 +303,6 @@ def _gen_endpoint(self, name, obj, description, version=1, schema=[]): endpoint_object = CustomQueryObject(query=obj, description=description,) - _schema = schema if schema is not None else [] return { "name": name, "version": version, diff --git a/tabpy/utils/user_management.py b/tabpy/utils/tabpy_user.py similarity index 62% rename from tabpy/utils/user_management.py rename to tabpy/utils/tabpy_user.py index 41326280..065d0298 100755 --- a/tabpy/utils/user_management.py +++ b/tabpy/utils/tabpy_user.py @@ -1,8 +1,22 @@ """ Utility for managing user names and passwords for TabPy. +For more information about how to configure and use authentication for +TabPy read the documentation at https://github.com/tableau/TabPy. + +Usage: + tabpy-user add (-u NAME | --user ) [-p PWD | --password PWD] (-f FILE | --pwdfile FILE) + tabpy-user update (-u NAME | --user ) [-p PWD | --password PWD] (-f FILE | --pwdfile FILE) + tabpy-user -h | --help + +Options: + -h --help Show this screen. + -u NAME --username NAME Username to add to password file. + -p PWD --password PWD Password for the username. If not specified a + password will be generated. + -f FILE --pwdfile FILE Fully qualified path to passwords file. """ -from argparse import ArgumentParser +import docopt import logging import secrets from tabpy.tabpy_server.app.util import parse_pwd_file @@ -11,40 +25,6 @@ logger = logging.getLogger(__name__) -def build_cli_parser(): - parser = ArgumentParser( - description=__doc__, - epilog=""" - For more information about how to configure and - use authentication for TabPy read the documentation - at https://github.com/tableau/TabPy - """, - argument_default=None, - add_help=True, - ) - parser.add_argument("command", choices=["add", "update"], help="Command to execute") - parser.add_argument("-u", "--username", help="Username to add to passwords file") - parser.add_argument( - "-f", "--pwdfile", help="Fully qualified path to passwords file" - ) - parser.add_argument( - "-p", - "--password", - help=( - "Password for the username. If not specified a password will " - "be generated" - ), - ) - return parser - - -def check_args(args): - if (args.username is None) or (args.pwdfile is None): - return False - - return True - - def generate_password(pwd_len=16): # List of characters to generate password from. # We want to avoid to use similarly looking pairs like @@ -84,7 +64,7 @@ def store_passwords_file(pwdfile, credentials): def add_user(args, credentials): - username = args.username.lower() + username = args["--username"].lower() logger.info(f'Adding username "{username}"') if username in credentials: @@ -95,11 +75,11 @@ def add_user(args, credentials): ) return False - password = args.password + password = args["--password"] logger.info(f'Adding username "{username}" with password "{password}"...') credentials[username] = hash_password(username, password) - if store_passwords_file(args.pwdfile, credentials): + if store_passwords_file(args["--pwdfile"], credentials): logger.info(f'Added username "{username}" with password "{password}"') else: logger.info( @@ -108,7 +88,7 @@ def add_user(args, credentials): def update_user(args, credentials): - username = args.username.lower() + username = args["--username"].lower() logger.info(f'Updating username "{username}"') if username not in credentials: @@ -118,16 +98,16 @@ def update_user(args, credentials): ) return False - password = args.password + password = args["--password"] logger.info(f'Updating username "{username}" password to "{password}"') credentials[username] = hash_password(username, password) - return store_passwords_file(args.pwdfile, credentials) + return store_passwords_file(args["--pwdfile"], credentials) def process_command(args, credentials): - if args.command == "add": + if args["add"]: return add_user(args, credentials) - elif args.command == "update": + elif args["update"]: return update_user(args, credentials) else: logger.error(f'Unknown command "{args.command}"') @@ -137,18 +117,14 @@ def process_command(args, credentials): def main(): logging.basicConfig(level=logging.DEBUG, format="%(message)s") - parser = build_cli_parser() - args = parser.parse_args() - if not check_args(args): - parser.print_help() - return + args = docopt.docopt(__doc__) - succeeded, credentials = parse_pwd_file(args.pwdfile) - if not succeeded and args.command != "add": + succeeded, credentials = parse_pwd_file(args["--pwdfile"]) + if not succeeded and not args["add"]: return - if args.password is None: - args.password = generate_password() + if args["--password"] is None: + args["--password"] = generate_password() process_command(args, credentials) return diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 67be3c71..84c6cd5f 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -1,6 +1,5 @@ import os import unittest -from argparse import Namespace from tempfile import NamedTemporaryFile import tabpy from tabpy.tabpy_server.app.util import validate_cert @@ -23,10 +22,6 @@ def test_config_file_does_not_exist(self): self.assertEqual(app.settings["log_request_context"], False) self.assertEqual(app.settings["evaluate_timeout"], 30) - @patch( - "tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments", - return_value=Namespace(config=None), - ) @patch("tabpy.tabpy_server.app.app.TabPyState") @patch("tabpy.tabpy_server.app.app._get_state_from_file") @patch("tabpy.tabpy_server.app.app.PythonServiceHandler") @@ -39,7 +34,6 @@ def test_no_config_file( mock_psws, mock_management_util, mock_tabpy_state, - mock_parse_arguments, ): pkg_path = os.path.dirname(tabpy.__file__) obj_path = os.path.join(pkg_path, "tmp", "query_objects") @@ -58,10 +52,6 @@ def test_no_config_file( self.assertTrue(len(mock_management_util.mock_calls) > 0) mock_os.makedirs.assert_not_called() - @patch( - "tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments", - return_value=Namespace(config=None), - ) @patch("tabpy.tabpy_server.app.app.TabPyState") @patch("tabpy.tabpy_server.app.app._get_state_from_file") @patch("tabpy.tabpy_server.app.app.PythonServiceHandler") @@ -74,7 +64,6 @@ def test_no_state_ini_file_or_state_dir( mock_psws, mock_management_util, mock_tabpy_state, - mock_parse_arguments, ): TabPyApp(None) self.assertEqual(len(mock_os.makedirs.mock_calls), 1) @@ -88,7 +77,6 @@ def tearDown(self): os.remove(self.config_file.name) self.config_file = None - @patch("tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments") @patch("tabpy.tabpy_server.app.app.TabPyState") @patch("tabpy.tabpy_server.app.app._get_state_from_file") @patch("tabpy.tabpy_server.app.app.PythonServiceHandler") @@ -101,7 +89,6 @@ def test_config_file_present( mock_psws, mock_management_util, mock_tabpy_state, - mock_parse_arguments, ): self.assertTrue(self.config_file is not None) config_file = self.config_file @@ -112,7 +99,6 @@ def test_config_file_present( ) config_file.close() - mock_parse_arguments.return_value = Namespace(config=config_file.name) mock_os.path.realpath.return_value = "bar" mock_os.environ = {"TABPY_PORT": "1234"} diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index 83f98628..4f6e9785 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -2,22 +2,14 @@ import os import tempfile -from argparse import Namespace from tabpy.tabpy_server.app.app import TabPyApp from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase -from unittest.mock import patch class TestEndpointHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): - cls.patcher = patch( - "tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments", - return_value=Namespace(config=None), - ) - cls.patcher.start() - prefix = "__TestEndpointHandlerWithAuth_" # create password file cls.pwd_file = tempfile.NamedTemporaryFile( @@ -62,7 +54,6 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - cls.patcher.stop() os.remove(cls.pwd_file.name) os.remove(cls.state_file.name) os.remove(cls.config_file.name) diff --git a/tests/unit/server_tests/test_endpoints_handler.py b/tests/unit/server_tests/test_endpoints_handler.py index 61a78c92..6ae00fc7 100755 --- a/tests/unit/server_tests/test_endpoints_handler.py +++ b/tests/unit/server_tests/test_endpoints_handler.py @@ -2,22 +2,14 @@ import os import tempfile -from argparse import Namespace from tabpy.tabpy_server.app.app import TabPyApp from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase -from unittest.mock import patch class TestEndpointsHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): - cls.patcher = patch( - "tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments", - return_value=Namespace(config=None), - ) - cls.patcher.start() - prefix = "__TestEndpointsHandlerWithAuth_" # create password file cls.pwd_file = tempfile.NamedTemporaryFile( @@ -62,7 +54,6 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - cls.patcher.stop() os.remove(cls.pwd_file.name) os.remove(cls.state_file.name) os.remove(cls.config_file.name) diff --git a/tests/unit/server_tests/test_evaluation_plane_handler.py b/tests/unit/server_tests/test_evaluation_plane_handler.py index 16979764..b2ef3473 100755 --- a/tests/unit/server_tests/test_evaluation_plane_handler.py +++ b/tests/unit/server_tests/test_evaluation_plane_handler.py @@ -6,18 +6,11 @@ from tabpy.tabpy_server.app.app import TabPyApp from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase -from unittest.mock import patch class TestEvaluationPlainHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): - cls.patcher = patch( - "tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments", - return_value=Namespace(config=None), - ) - cls.patcher.start() - prefix = "__TestEvaluationPlainHandlerWithAuth_" # create password file cls.pwd_file = tempfile.NamedTemporaryFile( @@ -85,7 +78,6 @@ def setUpClass(cls): @classmethod def tearDownClass(cls): - cls.patcher.stop() os.remove(cls.pwd_file.name) os.remove(cls.state_file.name) os.remove(cls.config_file.name) diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index ece2eea5..9132c2fe 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -1,11 +1,9 @@ -from argparse import Namespace import json import os from tabpy.tabpy_server.app.app import TabPyApp from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters import tempfile from tornado.testing import AsyncHTTPTestCase -from unittest.mock import patch def _create_expected_info_response(settings, tabpy_state): @@ -20,18 +18,6 @@ def _create_expected_info_response(settings, tabpy_state): class TestServiceInfoHandlerDefault(AsyncHTTPTestCase): - @classmethod - def setUpClass(cls): - cls.patcher = patch( - "tabpy.tabpy_server.app.app.TabPyApp._parse_cli_arguments", - return_value=Namespace(config=None), - ) - cls.patcher.start() - - @classmethod - def tearDownClass(cls): - cls.patcher.stop() - def get_app(self): self.app = TabPyApp() return self.app._create_tornado_web_app() From 36c59ba9d6b9386d277cd5bcdd0e8ae188b3905a Mon Sep 17 00:00:00 2001 From: Brennan Bugbee Date: Tue, 24 Dec 2019 17:25:23 -0500 Subject: [PATCH 13/61] Change regex, add remove method, and edit qeury_timeout (#375) * Added Client.remove method to delete deployed model * Fix bug for query_timeout types --- tabpy/tabpy_tools/client.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/tabpy/tabpy_tools/client.py b/tabpy/tabpy_tools/client.py index 0bf43522..96c2c267 100755 --- a/tabpy/tabpy_tools/client.py +++ b/tabpy/tabpy_tools/client.py @@ -26,7 +26,7 @@ def _check_endpoint_type(name): def _check_hostname(name): _check_endpoint_type(name) - hostname_checker = compile(r"^http(s)?://[a-zA-Z0-9-_\.]+(/)?(:[0-9]+)?(/)?$") + hostname_checker = compile(r"^^http(s)?://[\w.-]+(/)?(:\d+)?(/)?$") if not hostname_checker.match(name): raise ValueError( @@ -77,10 +77,10 @@ def __init__(self, endpoint, query_timeout=1000): service_client = ServiceClient(self._endpoint, network_wrapper) self._service = RESTServiceClient(service_client) - if query_timeout is not None and query_timeout > 0: - self.query_timeout = query_timeout + if type(query_timeout) in (int, float) and query_timeout > 0: + self._service.query_timeout = query_timeout else: - self.query_timeout = 0.0 + self._service.query_timeout = 0.0 def __repr__(self): return ( @@ -123,12 +123,13 @@ def get_status(self): @property def query_timeout(self): - """The timeout for queries in seconds.""" + """The timeout for queries in milliseconds.""" return self._service.query_timeout @query_timeout.setter def query_timeout(self, value): - self._service.query_timeout = value + if type(value) in (int, float) and value > 0: + self._service.query_timeout = value def query(self, name, *args, **kwargs): """Query an endpoint. @@ -248,7 +249,16 @@ def deploy(self, name, obj, description="", schema=None, override=False): self._wait_for_endpoint_deployment(obj["name"], obj["version"]) - def _gen_endpoint(self, name, obj, description, version=1, schema=[]): + def remove(self, name): + '''Removes an endpoint dict. + + Parameters + ---------- + name : str + Endpoint name to remove''' + self._service.remove_endpoint(name) + + def _gen_endpoint(self, name, obj, description, version=1, schema=None): """Generates an endpoint dict. Parameters From 665f1e1d23fa6b916050c91e39519a322ad8ca59 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Tue, 24 Dec 2019 14:56:58 -0800 Subject: [PATCH 14/61] Update CHANGELOG --- .vscode/settings.json | 2 +- CHANGELOG | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 4a047b81..e27fae0a 100755 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -18,5 +18,5 @@ "python.testing.unittestEnabled": false, "python.testing.nosetestsEnabled": false, "python.testing.pytestEnabled": true, - "python.linting.pep8Enabled": true + "python.linting.pycodestyleEnabled": true } \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG index 59b44e46..00641b9a 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -6,11 +6,12 @@ - Minor code improvements. - Documentation updates with referencing Tableau Help pages. -- +- Added Client.remove() method for deleting deployed models. ### Bug Fixes - Fixed failing Ctrl+C handler. +- Fixed query_timeout bug. ## v0.8.9 From a5bd2d1d9d83c9f2edb910efa05502f502d57bb0 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Thu, 26 Dec 2019 14:58:15 -0800 Subject: [PATCH 15/61] Fix missing script result/return bug --- ...veralls_codestyle.yml => pull_request.yml} | 0 .github/workflows/push.yml | 108 +++++++++++++++++ CHANGELOG | 2 + setup.py | 1 + .../handlers/evaluation_plane_handler.py | 5 +- tests/integration/test_evaluate.py | 109 ++++++++++++++++++ 6 files changed, 224 insertions(+), 1 deletion(-) rename .github/workflows/{test_coveralls_codestyle.yml => pull_request.yml} (100%) create mode 100755 .github/workflows/push.yml create mode 100755 tests/integration/test_evaluate.py diff --git a/.github/workflows/test_coveralls_codestyle.yml b/.github/workflows/pull_request.yml similarity index 100% rename from .github/workflows/test_coveralls_codestyle.yml rename to .github/workflows/pull_request.yml diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml new file mode 100755 index 00000000..4f91b6ea --- /dev/null +++ b/.github/workflows/push.yml @@ -0,0 +1,108 @@ +name: CI + +on: [pull_request] + +jobs: + ubuntu-build: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.6, 3.7] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Lint with flake8 + run: | + pip install flake8 + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Python Style Checker + uses: andymckay/pycodestyle-action@0.1.3 + + - name: Test with pytest + run: | + pip install pytest pytest-cov coveralls coverage==4.5.4 + pytest tests/unit --cov=tabpy --cov-append + pytest tests/integration --cov=tabpy --cov-append + coveralls + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} + + - name: Markdownlint + uses: nosborn/github-action-markdown-cli@v1.1.1 + with: + files: . + + windows-build: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.7] + os: [windows-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Test with pytest + run: | + pip install pytest pytest-cov coveralls + pytest tests/unit + pytest tests/integration + + mac-build: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.7] + os: [macos-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + + - name: Test with pytest + run: | + pip install pytest pytest-cov coveralls + pytest tests/unit + pytest tests/integration + \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG index 00641b9a..eed5b36b 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -12,6 +12,8 @@ - Fixed failing Ctrl+C handler. - Fixed query_timeout bug. +- Fixed None in result collection bug. +- Fixed script evaluation with missing result/return bug. ## v0.8.9 diff --git a/setup.py b/setup.py index a3418553..70c846e0 100755 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ def read(fname): "pyopenssl", "python-dateutil", "requests", + "simplejson", "singledispatch", "six", "tornado", diff --git a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py index f63458b9..384529f7 100644 --- a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py @@ -97,7 +97,10 @@ def post(self): self.error_out(408, self._error_message_timeout) return - self.write(simplejson.dumps(result, ignore_nan=True)) + if result is not None: + self.write(simplejson.dumps(result, ignore_nan=True)) + else: + self.write("null") self.finish() except Exception as e: diff --git a/tests/integration/test_evaluate.py b/tests/integration/test_evaluate.py new file mode 100755 index 00000000..ade8600a --- /dev/null +++ b/tests/integration/test_evaluate.py @@ -0,0 +1,109 @@ +""" +Script evaluation tests. +""" + +import integ_test_base +import json + + +class TestEvaluate(integ_test_base.IntegTestBase): + def test_single_value_returned(self): + payload = """ + { + "data": { "_arg1": 2, "_arg2": 40 }, + "script": + "return _arg1 + _arg2" + } + """ + headers = { + "Content-Type": "application/json", + } + + conn = self._get_connection() + conn.request("POST", "/evaluate", payload, headers) + response = conn.getresponse() + result = response.read().decode("utf-8") + + self.assertEqual(200, response.status) + self.assertEqual("42", result) + + def test_collection_returned(self): + payload = """ + { + "data": { "_arg1": [2, 3], "_arg2": [40, 0.1415926] }, + "script": + "return [x + y for x, y in zip(_arg1, _arg2)]" + } + """ + headers = { + "Content-Type": "application/json", + } + + conn = self._get_connection() + conn.request("POST", "/evaluate", payload, headers) + response = conn.getresponse() + result = response.read().decode("utf-8") + + self.assertEqual(200, response.status) + self.assertEqual("[42, 3.1415926]", result) + + def test_none_returned(self): + payload = """ + { + "data": { "_arg1": 2, "_arg2": 40 }, + "script": + "return None" + } + """ + headers = { + "Content-Type": "application/json", + } + + conn = self._get_connection() + conn.request("POST", "/evaluate", payload, headers) + response = conn.getresponse() + result = response.read().decode("utf-8") + + self.assertEqual(200, response.status) + self.assertEqual("null", result) + + def test_nothing_returned(self): + payload = """ + { + "data": { "_arg1": [2], "_arg2": [40] }, + "script": + "res = [x + y for x, y in zip(_arg1, _arg2)]" + } + """ + headers = { + "Content-Type": "application/json", + } + + conn = self._get_connection() + conn.request("POST", "/evaluate", payload, headers) + response = conn.getresponse() + result = response.read().decode("utf-8") + + self.assertEqual(200, response.status) + self.assertEqual("null", result) + + def test_syntax_error(self): + payload = """ + { + "data": { "_arg1": [2], "_arg2": [40] }, + "script": + "% ^ !! return Nothing" + } + """ + headers = { + "Content-Type": "application/json", + } + + conn = self._get_connection() + conn.request("POST", "/evaluate", payload, headers) + response = conn.getresponse() + result = json.loads(response.read().decode("utf-8")) + + self.assertEqual(500, response.status) + self.assertEqual("Error processing script", result["message"]) + self.assertTrue(result["info"].startswith("SyntaxError")) From 586eaccb2d5691ea55bd664fea7006fb787f73a8 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Thu, 26 Dec 2019 15:04:02 -0800 Subject: [PATCH 16/61] Fix github workflow for push --- .github/workflows/push.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4f91b6ea..2c7b091b 100755 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -1,6 +1,6 @@ name: CI -on: [pull_request] +on: [push] jobs: ubuntu-build: From 47aa6e0781b3774350a4e9db1d6c3b5cfda687e1 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Tue, 7 Jan 2020 13:52:24 -0800 Subject: [PATCH 17/61] Code improvement: app._parse_config (#391) --- .github/workflows/pull_request.yml | 3 - .github/workflows/push.yml | 3 - tabpy/tabpy_server/app/app.py | 171 ++++++++---------- tests/integration/test_auth.py | 5 + .../test_custom_evaluate_timeout.py | 5 + tests/integration/test_url.py | 5 + 6 files changed, 90 insertions(+), 102 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 4f91b6ea..4ca8032f 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -33,9 +33,6 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Python Style Checker - uses: andymckay/pycodestyle-action@0.1.3 - - name: Test with pytest run: | pip install pytest pytest-cov coveralls coverage==4.5.4 diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 2c7b091b..d3f88810 100755 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -33,9 +33,6 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Python Style Checker - uses: andymckay/pycodestyle-action@0.1.3 - - name: Test with pytest run: | pip install pytest pytest-cov coveralls coverage==4.5.4 diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 630f0f50..6c88a129 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -162,6 +162,34 @@ def try_exit(self): return application + def _set_parameter(self, parser, settings_key, config_key, default_val): + key_is_set = False + + if ( + config_key is not None + and parser.has_section("TabPy") + and parser.has_option("TabPy", config_key) + ): + self.settings[settings_key] = parser.get("TabPy", config_key) + key_is_set = True + logger.debug( + f"Parameter {settings_key} set to " + f'"{self.settings[settings_key]}" ' + "from config file or environment variable" + ) + + if not key_is_set and default_val is not None: + self.settings[settings_key] = default_val + key_is_set = True + logger.debug( + f"Parameter {settings_key} set to " + f'"{self.settings[settings_key]}" ' + "from default value" + ) + + if not key_is_set: + logger.debug(f"Parameter {settings_key} is not set") + def _parse_config(self, config_file): """Provide consistent mechanism for pulling in configuration. @@ -189,6 +217,8 @@ def _parse_config(self, config_file): self.python_service = None self.credentials = {} + pkg_path = os.path.dirname(tabpy.__file__) + parser = configparser.ConfigParser(os.environ) if os.path.isfile(config_file): @@ -200,44 +230,29 @@ def _parse_config(self, config_file): "using default settings." ) - def set_parameter(settings_key, config_key, default_val=None): - key_is_set = False - - if ( - config_key is not None - and parser.has_section("TabPy") - and parser.has_option("TabPy", config_key) - ): - self.settings[settings_key] = parser.get("TabPy", config_key) - key_is_set = True - logger.debug( - f"Parameter {settings_key} set to " - f'"{self.settings[settings_key]}" ' - "from config file or environment variable" - ) - - if not key_is_set and default_val is not None: - self.settings[settings_key] = default_val - key_is_set = True - logger.debug( - f"Parameter {settings_key} set to " - f'"{self.settings[settings_key]}" ' - "from default value" - ) - - if not key_is_set: - logger.debug(f"Parameter {settings_key} is not set") - - set_parameter( - SettingsParameters.Port, ConfigParameters.TABPY_PORT, default_val=9004 - ) - set_parameter(SettingsParameters.ServerVersion, None, default_val=__version__) - - set_parameter( - SettingsParameters.EvaluateTimeout, - ConfigParameters.TABPY_EVALUATE_TIMEOUT, - default_val=30, - ) + settings_parameters = [ + (SettingsParameters.Port, ConfigParameters.TABPY_PORT, 9004), + (SettingsParameters.ServerVersion, None, __version__), + (SettingsParameters.EvaluateTimeout, ConfigParameters.TABPY_EVALUATE_TIMEOUT, 30), + (SettingsParameters.UploadDir, ConfigParameters.TABPY_QUERY_OBJECT_PATH, + os.path.join(pkg_path, "tmp", "query_objects")), + (SettingsParameters.TransferProtocol, ConfigParameters.TABPY_TRANSFER_PROTOCOL, + "http"), + (SettingsParameters.CertificateFile, ConfigParameters.TABPY_CERTIFICATE_FILE, + None), + (SettingsParameters.KeyFile, ConfigParameters.TABPY_KEY_FILE, None), + (SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, + os.path.join(pkg_path, "tabpy_server")), + (SettingsParameters.StaticPath, ConfigParameters.TABPY_STATIC_PATH, "./"), + (ConfigParameters.TABPY_PWD_FILE, ConfigParameters.TABPY_PWD_FILE, None), + (SettingsParameters.LogRequestContext, ConfigParameters.TABPY_LOG_DETAILS, + "false"), + (SettingsParameters.MaxRequestSizeInMb, ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB, + 100), + ] + + for setting, parameter, default_val in settings_parameters: + self._set_parameter(parser, setting, parameter, default_val) try: self.settings[SettingsParameters.EvaluateTimeout] = float( @@ -250,66 +265,27 @@ def set_parameter(settings_key, config_key, default_val=None): ) self.settings[SettingsParameters.EvaluateTimeout] = 30 - pkg_path = os.path.dirname(tabpy.__file__) - set_parameter( - SettingsParameters.UploadDir, - ConfigParameters.TABPY_QUERY_OBJECT_PATH, - default_val=os.path.join(pkg_path, "tmp", "query_objects"), - ) if not os.path.exists(self.settings[SettingsParameters.UploadDir]): os.makedirs(self.settings[SettingsParameters.UploadDir]) # set and validate transfer protocol - set_parameter( - SettingsParameters.TransferProtocol, - ConfigParameters.TABPY_TRANSFER_PROTOCOL, - default_val="http", - ) self.settings[SettingsParameters.TransferProtocol] = self.settings[ SettingsParameters.TransferProtocol ].lower() - set_parameter( - SettingsParameters.CertificateFile, ConfigParameters.TABPY_CERTIFICATE_FILE - ) - set_parameter(SettingsParameters.KeyFile, ConfigParameters.TABPY_KEY_FILE) self._validate_transfer_protocol_settings() # if state.ini does not exist try and create it - remove # last dependence on batch/shell script - set_parameter( - SettingsParameters.StateFilePath, - ConfigParameters.TABPY_STATE_PATH, - default_val=os.path.join(pkg_path, "tabpy_server"), - ) self.settings[SettingsParameters.StateFilePath] = os.path.realpath( os.path.normpath( os.path.expanduser(self.settings[SettingsParameters.StateFilePath]) ) ) - state_file_dir = self.settings[SettingsParameters.StateFilePath] - state_file_path = os.path.join(state_file_dir, "state.ini") - if not os.path.isfile(state_file_path): - state_file_template_path = os.path.join( - pkg_path, "tabpy_server", "state.ini.template" - ) - logger.debug( - f"File {state_file_path} not found, creating from " - f"template {state_file_template_path}..." - ) - shutil.copy(state_file_template_path, state_file_path) - - logger.info(f"Loading state from state file {state_file_path}") - tabpy_state = _get_state_from_file(state_file_dir) - self.tabpy_state = TabPyState(config=tabpy_state, settings=self.settings) + state_config, self.tabpy_state = self._build_tabpy_state() self.python_service = PythonServiceHandler(PythonService()) self.settings["compress_response"] = True - set_parameter( - SettingsParameters.StaticPath, - ConfigParameters.TABPY_STATIC_PATH, - default_val="./", - ) self.settings[SettingsParameters.StaticPath] = os.path.abspath( self.settings[SettingsParameters.StaticPath] ) @@ -319,11 +295,10 @@ def set_parameter(settings_key, config_key, default_val=None): ) # Set subdirectory from config if applicable - if tabpy_state.has_option("Service Info", "Subdirectory"): - self.subdirectory = "/" + tabpy_state.get("Service Info", "Subdirectory") + if state_config.has_option("Service Info", "Subdirectory"): + self.subdirectory = "/" + state_config.get("Service Info", "Subdirectory") # If passwords file specified load credentials - set_parameter(ConfigParameters.TABPY_PWD_FILE, ConfigParameters.TABPY_PWD_FILE) if ConfigParameters.TABPY_PWD_FILE in self.settings: if not self._parse_pwd_file(): msg = ( @@ -340,11 +315,6 @@ def set_parameter(settings_key, config_key, default_val=None): features = self._get_features() self.settings[SettingsParameters.ApiVersions] = {"v1": {"features": features}} - set_parameter( - SettingsParameters.LogRequestContext, - ConfigParameters.TABPY_LOG_DETAILS, - default_val="false", - ) self.settings[SettingsParameters.LogRequestContext] = ( self.settings[SettingsParameters.LogRequestContext].lower() != "false" ) @@ -355,16 +325,6 @@ def set_parameter(settings_key, config_key, default_val=None): ) logger.info(f"Call context logging is {call_context_state}") - set_parameter( - SettingsParameters.MaxRequestSizeInMb, - ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB, - default_val=100, - ) - - set_parameter(SettingsParameters.MaxRequestSizeInMb, - ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB, - default_val=100) - def _validate_transfer_protocol_settings(self): if SettingsParameters.TransferProtocol not in self.settings: msg = "Missing transfer protocol information." @@ -436,3 +396,22 @@ def _get_features(self): } return features + + def _build_tabpy_state(self): + pkg_path = os.path.dirname(tabpy.__file__) + state_file_dir = self.settings[SettingsParameters.StateFilePath] + state_file_path = os.path.join(state_file_dir, "state.ini") + if not os.path.isfile(state_file_path): + state_file_template_path = os.path.join( + pkg_path, "tabpy_server", "state.ini.template" + ) + logger.debug( + f"File {state_file_path} not found, creating from " + f"template {state_file_template_path}..." + ) + shutil.copy(state_file_template_path, state_file_path) + + logger.info(f"Loading state from state file {state_file_path}") + tabpy_state = _get_state_from_file(state_file_dir) + return tabpy_state, TabPyState(config=tabpy_state, settings=self.settings) + \ No newline at end of file diff --git a/tests/integration/test_auth.py b/tests/integration/test_auth.py index a3d495bb..4b1a4bca 100755 --- a/tests/integration/test_auth.py +++ b/tests/integration/test_auth.py @@ -40,6 +40,11 @@ def test_invalid_password(self): self.assertEqual(401, res.status) def test_invalid_username(self): + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + # self.set_delete_temp_folder(False) + headers = { "Content-Type": "application/json", "TabPy-Client": "Integration tests for Auth", diff --git a/tests/integration/test_custom_evaluate_timeout.py b/tests/integration/test_custom_evaluate_timeout.py index 3a38e21d..2150c551 100644 --- a/tests/integration/test_custom_evaluate_timeout.py +++ b/tests/integration/test_custom_evaluate_timeout.py @@ -6,6 +6,11 @@ def _get_evaluate_timeout(self) -> str: return "5" def test_custom_evaluate_timeout_with_script(self): + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + # self.set_delete_temp_folder(False) + payload = """ { "data": { "_arg1": 1 }, diff --git a/tests/integration/test_url.py b/tests/integration/test_url.py index 070296cb..62fd3516 100755 --- a/tests/integration/test_url.py +++ b/tests/integration/test_url.py @@ -7,6 +7,11 @@ class TestURL(integ_test_base.IntegTestBase): def test_notexistent_url(self): + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + # self.set_delete_temp_folder(False) + conn = self._get_connection() conn.request("GET", "/unicorn") res = conn.getresponse() From ddabcc46e6d7d08291e61f937c20ae596ae7f5e1 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Tue, 7 Jan 2020 13:55:45 -0800 Subject: [PATCH 18/61] Update app.py --- tabpy/tabpy_server/app/app.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 6c88a129..ac98ed65 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -414,4 +414,3 @@ def _build_tabpy_state(self): logger.info(f"Loading state from state file {state_file_path}") tabpy_state = _get_state_from_file(state_file_dir) return tabpy_state, TabPyState(config=tabpy_state, settings=self.settings) - \ No newline at end of file From 06a98ee085756f45a73f4f10efe3dcd9171183f2 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Fri, 10 Jan 2020 17:16:01 -0800 Subject: [PATCH 19/61] v0.8.10 (#392) * Collect test coverage with scrutinizer instead of coveralls * Restore coverage collecting with coveralls * Update scrutinizer settings * Add support for Python 3.8 * Fix static page and add unit test for it * Delete obsolete test --- .github/workflows/pull_request.yml | 25 ++++---- .github/workflows/push.yml | 25 ++++---- .scrutinizer.yml | 41 ++++++++----- CHANGELOG | 6 +- CONTRIBUTING.md | 11 +--- requirements_dev.txt | 1 + requirements_test.txt | 3 + setup.py | 2 +- tabpy/tabpy_server/app/app.py | 24 +++++++- tabpy/tabpy_server/static/index.html | 61 ++++++++++++++++--- tests/integration/test_url.py | 7 +++ tests/unit/server_tests/__init__.py | 0 .../server_tests/test_endpoint_handler.py | 21 +++++++ .../test_evaluation_plane_handler.py | 4 -- tests/unit/tools_tests/test_rest.py | 4 +- 15 files changed, 167 insertions(+), 68 deletions(-) create mode 100755 requirements_dev.txt create mode 100755 requirements_test.txt delete mode 100644 tests/unit/server_tests/__init__.py diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 4ca8032f..aa2f2cc3 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: - python-version: [3.6, 3.7] + python-version: [3.6, 3.7, 3.8] os: [ubuntu-latest] steps: @@ -24,10 +24,11 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install -r requirements_test.txt + pip install -r requirements_dev.txt - name: Lint with flake8 run: | - pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide @@ -35,11 +36,9 @@ jobs: - name: Test with pytest run: | - pip install pytest pytest-cov coveralls coverage==4.5.4 - pytest tests/unit --cov=tabpy --cov-append - pytest tests/integration --cov=tabpy --cov-append + pytest tests --cov=tabpy coveralls - env: + env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - name: Markdownlint @@ -53,7 +52,7 @@ jobs: strategy: matrix: - python-version: [3.7] + python-version: [3.7, 3.8] os: [windows-latest] steps: @@ -68,12 +67,11 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install -r requirements_test.txt - name: Test with pytest run: | - pip install pytest pytest-cov coveralls - pytest tests/unit - pytest tests/integration + pytest tests mac-build: name: ${{ matrix.python-version }} on ${{ matrix.os }} @@ -81,7 +79,7 @@ jobs: strategy: matrix: - python-version: [3.7] + python-version: [3.7, 3.8] os: [macos-latest] steps: @@ -96,10 +94,9 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install -r requirements_test.txt - name: Test with pytest run: | - pip install pytest pytest-cov coveralls - pytest tests/unit - pytest tests/integration + pytest tests \ No newline at end of file diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index d3f88810..08874122 100755 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -9,7 +9,7 @@ jobs: strategy: matrix: - python-version: [3.6, 3.7] + python-version: [3.6, 3.7, 3.8] os: [ubuntu-latest] steps: @@ -24,10 +24,11 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install -r requirements_test.txt + pip install -r requirements_dev.txt - name: Lint with flake8 run: | - pip install flake8 # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide @@ -35,11 +36,9 @@ jobs: - name: Test with pytest run: | - pip install pytest pytest-cov coveralls coverage==4.5.4 - pytest tests/unit --cov=tabpy --cov-append - pytest tests/integration --cov=tabpy --cov-append + pytest tests --cov=tabpy coveralls - env: + env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - name: Markdownlint @@ -53,7 +52,7 @@ jobs: strategy: matrix: - python-version: [3.7] + python-version: [3.7, 3.8] os: [windows-latest] steps: @@ -68,12 +67,11 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install -r requirements_test.txt - name: Test with pytest run: | - pip install pytest pytest-cov coveralls - pytest tests/unit - pytest tests/integration + pytest tests mac-build: name: ${{ matrix.python-version }} on ${{ matrix.os }} @@ -81,7 +79,7 @@ jobs: strategy: matrix: - python-version: [3.7] + python-version: [3.7, 3.8] os: [macos-latest] steps: @@ -96,10 +94,9 @@ jobs: run: | python -m pip install --upgrade pip pip install -r requirements.txt + pip install -r requirements_test.txt - name: Test with pytest run: | - pip install pytest pytest-cov coveralls - pytest tests/unit - pytest tests/integration + pytest tests \ No newline at end of file diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 22572143..01d763a1 100755 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,18 +1,31 @@ build: - environment: - python: 3.6 - nodes: - analysis: - project_setup: - override: - - pip install sklearn pandas numpy textblob nltk scipy - tests: - override: - - py-scrutinizer-run - - - command: pylint-run - use_website_config: true - tests: true + environment: + python: 3.6 + tests: + before: + - pip install pytest pytest-cov coverage + - pip install -r requirements.txt + nodes: + analysis: + project_setup: + override: + - pip install sklearn pandas numpy textblob nltk scipy + tests: + override: + - py-scrutinizer-run + - command: pylint-run + use_website_config: true + coverage: + tests: + before: + - pip install pytest pytest-cov coverage coveralls + - pip install -r requirements.txt + override: + - command: pytest tests --cov=tabpy + coverage: + file: '.coverage' + config_file: '.coveragerc' + format: 'py-cc' checks: python: code_rating: true diff --git a/CHANGELOG b/CHANGELOG index eed5b36b..98a05232 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,19 +1,21 @@ # Changelog -## v0.8.9 +## v0.8.10 ### Improvements -- Minor code improvements. +- TabPy works with Python 3.8 now. - Documentation updates with referencing Tableau Help pages. - Added Client.remove() method for deleting deployed models. + ### Bug Fixes - Fixed failing Ctrl+C handler. - Fixed query_timeout bug. - Fixed None in result collection bug. - Fixed script evaluation with missing result/return bug. +- Fixed startup failure on Windows for Python 3.8. ## v0.8.9 diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ba332aea..d51507fa 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -34,13 +34,6 @@ be able to work on TabPy changes: - Python 3.6 or 3.7: - To see which version of Python you have installed, run `python --version`. - git -- TabPy repo: - - Get the latest TabPy repository with - `git clone https://github.com/tableau/TabPy.git`. - - Create a new branch for your changes. - - When changes are ready push them on github and create merge request. -- PIP packages - install all with - `pip install pytest flake8 twine coverage --upgrade` command - Node.js for npm packages - install from . - NPM packages - install all with `npm install markdown-toc markdownlint` command. @@ -62,6 +55,8 @@ be able to work on TabPy changes: ```sh python -m pip install --upgrade pip pip install -r requirements.txt + pip install -r requirements_dev.txt + pip install -r requirements_test.txt ``` ## Tests @@ -94,7 +89,7 @@ You can run unit tests to collect code coverage data. To do so run `pytest` either for server or tools test, or even combined: ```sh -pytest tests --cov=tabpy-server/tabpy_server --cov=tabpy-tools/tabpy_tools --cov-append +pytest tests --cov=tabpy ``` ## TabPy in Python Virtual Environment diff --git a/requirements_dev.txt b/requirements_dev.txt new file mode 100755 index 00000000..1ae3ec61 --- /dev/null +++ b/requirements_dev.txt @@ -0,0 +1 @@ +flake8 \ No newline at end of file diff --git a/requirements_test.txt b/requirements_test.txt new file mode 100755 index 00000000..2bbfacd9 --- /dev/null +++ b/requirements_test.txt @@ -0,0 +1,3 @@ +coveralls +pytest +pytest-cov \ No newline at end of file diff --git a/setup.py b/setup.py index 70c846e0..cdafa7e0 100755 --- a/setup.py +++ b/setup.py @@ -79,7 +79,7 @@ def read(fname): "singledispatch", "six", "tornado", - "urllib3<1.25,>=1.21.1", + "urllib3", ], entry_points={ "console_scripts": [ diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index ac98ed65..0ee807ad 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -6,6 +6,7 @@ import os import shutil import signal +import sys import tabpy.tabpy_server from tabpy.tabpy import __version__ from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters @@ -30,6 +31,25 @@ logger = logging.getLogger(__name__) +def _init_asyncio_patch(): + """ + Select compatible event loop for Tornado 5+. + As of Python 3.8, the default event loop on Windows is `proactor`, + however Tornado requires the old default "selector" event loop. + As Tornado has decided to leave this to users to set, MkDocs needs + to set it. See https://github.com/tornadoweb/tornado/issues/2608. + """ + if sys.platform.startswith("win") and sys.version_info >= (3, 8): + import asyncio + try: + from asyncio import WindowsSelectorEventLoopPolicy + except ImportError: + pass # Can't assign a policy which doesn't exist. + else: + if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy): + asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) + + class TabPyApp: """ TabPy application class for keeping context like settings, state, etc. @@ -113,6 +133,7 @@ def try_exit(self): ) # initialize Tornado application + _init_asyncio_patch() application = TabPyTornadoApp( [ # skip MainHandler to use StaticFileHandler .* page requests and @@ -243,7 +264,8 @@ def _parse_config(self, config_file): (SettingsParameters.KeyFile, ConfigParameters.TABPY_KEY_FILE, None), (SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, os.path.join(pkg_path, "tabpy_server")), - (SettingsParameters.StaticPath, ConfigParameters.TABPY_STATIC_PATH, "./"), + (SettingsParameters.StaticPath, ConfigParameters.TABPY_STATIC_PATH, + os.path.join(pkg_path, "tabpy_server", "static")), (ConfigParameters.TABPY_PWD_FILE, ConfigParameters.TABPY_PWD_FILE, None), (SettingsParameters.LogRequestContext, ConfigParameters.TABPY_LOG_DETAILS, "false"), diff --git a/tabpy/tabpy_server/static/index.html b/tabpy/tabpy_server/static/index.html index 7f2da21d..37088552 100644 --- a/tabpy/tabpy_server/static/index.html +++ b/tabpy/tabpy_server/static/index.html @@ -8,19 +8,64 @@ Tableau Python Server - + + - - - +

TabPy Server Info:

+

+

Deployed Models:

+

- \ No newline at end of file +

Useful links:

+ + + + diff --git a/tests/integration/test_url.py b/tests/integration/test_url.py index 62fd3516..0fa4c03c 100755 --- a/tests/integration/test_url.py +++ b/tests/integration/test_url.py @@ -17,3 +17,10 @@ def test_notexistent_url(self): res = conn.getresponse() self.assertEqual(404, res.status) + + def test_static_page(self): + conn = self._get_connection() + conn.request("GET", "/") + res = conn.getresponse() + + self.assertEqual(200, res.status) diff --git a/tests/unit/server_tests/__init__.py b/tests/unit/server_tests/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index 4f6e9785..40287371 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -1,5 +1,6 @@ import base64 import os +import sys import tempfile from tabpy.tabpy_server.app.app import TabPyApp @@ -7,9 +8,29 @@ from tornado.testing import AsyncHTTPTestCase +def _init_asyncio_patch(): + """ + Select compatible event loop for Tornado 5+. + As of Python 3.8, the default event loop on Windows is `proactor`, + however Tornado requires the old default "selector" event loop. + As Tornado has decided to leave this to users to set, MkDocs needs + to set it. See https://github.com/tornadoweb/tornado/issues/2608. + """ + if sys.platform.startswith("win") and sys.version_info >= (3, 8): + import asyncio + try: + from asyncio import WindowsSelectorEventLoopPolicy + except ImportError: + pass # Can't assign a policy which doesn't exist. + else: + if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy): + asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) + + class TestEndpointHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): + _init_asyncio_patch() prefix = "__TestEndpointHandlerWithAuth_" # create password file cls.pwd_file = tempfile.NamedTemporaryFile( diff --git a/tests/unit/server_tests/test_evaluation_plane_handler.py b/tests/unit/server_tests/test_evaluation_plane_handler.py index 408c6a3c..49b67dfb 100755 --- a/tests/unit/server_tests/test_evaluation_plane_handler.py +++ b/tests/unit/server_tests/test_evaluation_plane_handler.py @@ -130,10 +130,6 @@ def test_valid_creds_pass(self): ) self.assertEqual(200, response.code) - def test_null_request(self): - response = self.fetch("") - self.assertEqual(404, response.code) - def test_script_not_present(self): response = self.fetch( "/evaluate", diff --git a/tests/unit/tools_tests/test_rest.py b/tests/unit/tools_tests/test_rest.py index 59f74234..33a78bf2 100644 --- a/tests/unit/tools_tests/test_rest.py +++ b/tests/unit/tools_tests/test_rest.py @@ -18,14 +18,14 @@ def test_init_with_session(self): self.assertIs(session, rnw.session) def mock_response(self, status_code): - response = Mock(requests.Response()) + response = Mock(requests.Response) response.json.return_value = "json" response.status_code = status_code return response def setUp(self): - session = Mock(requests.session()) + session = Mock(requests.Session) session.get.return_value = self.mock_response(200) session.post.return_value = self.mock_response(200) session.put.return_value = self.mock_response(200) From 0703f68497a1bd86b7888aeb9584099a4739d590 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Fri, 10 Jan 2020 17:26:34 -0800 Subject: [PATCH 20/61] Dev cov (#394) * Collect test coverage with scrutinizer instead of coveralls * Restore coverage collecting with coveralls * Update scrutinizer settings * Add support for Python 3.8 * Fix static page and add unit test for it * Delete obsolete test * Restore scrutinizer configuration --- .scrutinizer.yml | 60 +++++++++++++++++++----------------------------- 1 file changed, 23 insertions(+), 37 deletions(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 01d763a1..a90a84a2 100755 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,37 +1,23 @@ -build: - environment: - python: 3.6 - tests: - before: - - pip install pytest pytest-cov coverage - - pip install -r requirements.txt - nodes: - analysis: - project_setup: - override: - - pip install sklearn pandas numpy textblob nltk scipy - tests: - override: - - py-scrutinizer-run - - command: pylint-run - use_website_config: true - coverage: - tests: - before: - - pip install pytest pytest-cov coverage coveralls - - pip install -r requirements.txt - override: - - command: pytest tests --cov=tabpy - coverage: - file: '.coverage' - config_file: '.coveragerc' - format: 'py-cc' -checks: - python: - code_rating: true - duplicate_code: true -filter: - excluded_paths: - - '*/test/*' - dependency_paths: - - 'lib/*' +build: + environment: + python: 3.8 + nodes: + analysis: + project_setup: + override: + - pip install sklearn pandas numpy textblob nltk scipy + tests: + override: + - py-scrutinizer-run + - command: pylint-run + use_website_config: true + tests: true +checks: + python: + code_rating: true + duplicate_code: true +filter: + excluded_paths: + - '*/test/*' + dependency_paths: + - 'lib/*' From 0f872e95844f3aab16a9742590bca3716ee5962a Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Fri, 10 Jan 2020 17:38:35 -0800 Subject: [PATCH 21/61] Linting as separate build step --- .github/workflows/pull_request.yml | 49 ++++++++++++++++++++++-------- .github/workflows/push.yml | 49 ++++++++++++++++++++++-------- 2 files changed, 72 insertions(+), 26 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index aa2f2cc3..2857b5bb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -27,13 +27,6 @@ jobs: pip install -r requirements_test.txt pip install -r requirements_dev.txt - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest run: | pytest tests --cov=tabpy @@ -41,11 +34,6 @@ jobs: env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - - name: Markdownlint - uses: nosborn/github-action-markdown-cli@v1.1.1 - with: - files: . - windows-build: name: ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -99,4 +87,39 @@ jobs: - name: Test with pytest run: | pytest tests - \ No newline at end of file + + sources-validation: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.7] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements_test.txt + pip install -r requirements_dev.txt + + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Markdownlint + uses: nosborn/github-action-markdown-cli@v1.1.1 + with: + files: . diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 08874122..f05755b7 100755 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -27,13 +27,6 @@ jobs: pip install -r requirements_test.txt pip install -r requirements_dev.txt - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest run: | pytest tests --cov=tabpy @@ -41,11 +34,6 @@ jobs: env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - - name: Markdownlint - uses: nosborn/github-action-markdown-cli@v1.1.1 - with: - files: . - windows-build: name: ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} @@ -99,4 +87,39 @@ jobs: - name: Test with pytest run: | pytest tests - \ No newline at end of file + + sources-validation: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.7] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements_test.txt + pip install -r requirements_dev.txt + + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Markdownlint + uses: nosborn/github-action-markdown-cli@v1.1.1 + with: + files: . From 58a27602c4ccb5af19ae1e215583ee27c6bfede1 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Fri, 10 Jan 2020 17:41:34 -0800 Subject: [PATCH 22/61] Restore scrutinizer configuration --- .scrutinizer.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index a90a84a2..993787f1 100755 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -9,7 +9,8 @@ build: tests: override: - py-scrutinizer-run - - command: pylint-run + - + command: pylint-run use_website_config: true tests: true checks: From 2f713b753cabbed0111352d6424eb72d83260438 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Fri, 10 Jan 2020 17:44:14 -0800 Subject: [PATCH 23/61] Update .scrutinizer.yml --- .scrutinizer.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 993787f1..6273afbf 100755 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -12,13 +12,12 @@ build: - command: pylint-run use_website_config: true - tests: true checks: - python: - code_rating: true - duplicate_code: true + python: + code_rating: true + duplicate_code: true filter: - excluded_paths: - - '*/test/*' - dependency_paths: - - 'lib/*' + excluded_paths: + - '*/test/*' + dependency_paths: + - 'lib/*' From 548b8e4eed23346a9e69e5d32e8c366b512b0646 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Fri, 10 Jan 2020 17:46:38 -0800 Subject: [PATCH 24/61] Update .scrutinizer.yml --- .scrutinizer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index 6273afbf..b848c193 100755 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,6 +1,6 @@ build: environment: - python: 3.8 + python: 3.6 nodes: analysis: project_setup: From 0ae345fbfaf0f0a7611e7ac5f2b16818edb7b3cd Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Fri, 10 Jan 2020 17:47:37 -0800 Subject: [PATCH 25/61] Restore scrutinizer configuration --- .scrutinizer.yml | 47 ++++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index b848c193..22572143 100755 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -1,23 +1,24 @@ -build: - environment: - python: 3.6 - nodes: - analysis: - project_setup: - override: - - pip install sklearn pandas numpy textblob nltk scipy - tests: - override: - - py-scrutinizer-run - - - command: pylint-run - use_website_config: true -checks: - python: - code_rating: true - duplicate_code: true -filter: - excluded_paths: - - '*/test/*' - dependency_paths: - - 'lib/*' +build: + environment: + python: 3.6 + nodes: + analysis: + project_setup: + override: + - pip install sklearn pandas numpy textblob nltk scipy + tests: + override: + - py-scrutinizer-run + - + command: pylint-run + use_website_config: true + tests: true +checks: + python: + code_rating: true + duplicate_code: true +filter: + excluded_paths: + - '*/test/*' + dependency_paths: + - 'lib/*' From 4f2ecd8531573179f647d7643cce805ae5c7a02d Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Fri, 10 Jan 2020 17:54:15 -0800 Subject: [PATCH 26/61] Update pull_request.yml --- .github/workflows/pull_request.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 2857b5bb..041f85fb 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -94,7 +94,7 @@ jobs: strategy: matrix: - python-version: [3.7] + python-version: [3.6] os: [ubuntu-latest] steps: From 9c5b7ccadaa6c3cb3e5f606fbf9fc86c690b7885 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Fri, 10 Jan 2020 18:43:12 -0800 Subject: [PATCH 27/61] Code style improvements --- .github/workflows/lint.yml | 40 ++++++ .github/workflows/pull_request.yml | 36 ------ .github/workflows/push.yml | 36 ------ .../handlers/evaluation_plane_handler.py | 114 +++++++++--------- tabpy/tabpy_server/management/state.py | 23 ++-- .../test_custom_evaluate_timeout.py | 10 +- .../server_tests/test_endpoint_handler.py | 22 +--- 7 files changed, 122 insertions(+), 159 deletions(-) create mode 100755 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100755 index 00000000..6dffd140 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,40 @@ +name: Lint + +on: [push, pull_request] + +jobs: + lint: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.7] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements_test.txt + pip install -r requirements_dev.txt + + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + + - name: Markdownlint + uses: nosborn/github-action-markdown-cli@v1.1.1 + with: + files: . diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 041f85fb..38d520d1 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -87,39 +87,3 @@ jobs: - name: Test with pytest run: | pytest tests - - sources-validation: - name: ${{ matrix.python-version }} on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - - strategy: - matrix: - python-version: [3.6] - os: [ubuntu-latest] - - steps: - - uses: actions/checkout@v1 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements_test.txt - pip install -r requirements_dev.txt - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Markdownlint - uses: nosborn/github-action-markdown-cli@v1.1.1 - with: - files: . diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index f05755b7..479d6957 100755 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -87,39 +87,3 @@ jobs: - name: Test with pytest run: | pytest tests - - sources-validation: - name: ${{ matrix.python-version }} on ${{ matrix.os }} - runs-on: ${{ matrix.os }} - - strategy: - matrix: - python-version: [3.7] - os: [ubuntu-latest] - - steps: - - uses: actions/checkout@v1 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements.txt - pip install -r requirements_test.txt - pip install -r requirements_dev.txt - - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - - name: Markdownlint - uses: nosborn/github-action-markdown-cli@v1.1.1 - with: - files: . diff --git a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py index 384529f7..390aff04 100644 --- a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py @@ -40,6 +40,64 @@ def initialize(self, executor, app): f"Timeout is set to {self.eval_timeout} s." ) + @gen.coroutine + def _post_impl(self): + body = json.loads(self.request.body.decode("utf-8")) + self.logger.log(logging.DEBUG, f"Processing POST request '{body}'...") + if "script" not in body: + self.error_out(400, "Script is empty.") + return + + # Transforming user script into a proper function. + user_code = body["script"] + arguments = None + arguments_str = "" + if "data" in body: + arguments = body["data"] + + if arguments is not None: + if not isinstance(arguments, dict): + self.error_out( + 400, "Script parameters need to be provided as a dictionary." + ) + return + args_in = sorted(arguments.keys()) + n = len(arguments) + if sorted('_arg'+str(i+1) for i in range(n)) == args_in: + arguments_str = ", " + ", ".join(args_in) + else: + self.error_out( + 400, + "Variables names should follow " + "the format _arg1, _arg2, _argN", + ) + return + + function_to_evaluate = f"def _user_script(tabpy{arguments_str}):\n" + for u in user_code.splitlines(): + function_to_evaluate += " " + u + "\n" + + self.logger.log( + logging.INFO, f"function to evaluate={function_to_evaluate}" + ) + + try: + result = yield self._call_subprocess(function_to_evaluate, arguments) + except ( + gen.TimeoutError, + requests.exceptions.ConnectTimeout, + requests.exceptions.ReadTimeout, + ): + self.logger.log(logging.ERROR, self._error_message_timeout) + self.error_out(408, self._error_message_timeout) + return + + if result is not None: + self.write(simplejson.dumps(result, ignore_nan=True)) + else: + self.write("null") + self.finish() + @gen.coroutine def post(self): if self.should_fail_with_not_authorized(): @@ -48,61 +106,7 @@ def post(self): self._add_CORS_header() try: - body = json.loads(self.request.body.decode("utf-8")) - if "script" not in body: - self.error_out(400, "Script is empty.") - return - - # Transforming user script into a proper function. - user_code = body["script"] - arguments = None - arguments_str = "" - if "data" in body: - arguments = body["data"] - - if arguments is not None: - if not isinstance(arguments, dict): - self.error_out( - 400, "Script parameters need to be provided as a dictionary." - ) - return - args_in = sorted(arguments.keys()) - n = len(arguments) - if sorted('_arg'+str(i+1) for i in range(n)) == args_in: - arguments_str = ", " + ", ".join(args_in) - else: - self.error_out( - 400, - "Variables names should follow " - "the format _arg1, _arg2, _argN", - ) - return - - function_to_evaluate = f"def _user_script(tabpy{arguments_str}):\n" - for u in user_code.splitlines(): - function_to_evaluate += " " + u + "\n" - - self.logger.log( - logging.INFO, f"function to evaluate={function_to_evaluate}" - ) - - try: - result = yield self._call_subprocess(function_to_evaluate, arguments) - except ( - gen.TimeoutError, - requests.exceptions.ConnectTimeout, - requests.exceptions.ReadTimeout, - ): - self.logger.log(logging.ERROR, self._error_message_timeout) - self.error_out(408, self._error_message_timeout) - return - - if result is not None: - self.write(simplejson.dumps(result, ignore_nan=True)) - else: - self.write("null") - self.finish() - + yield self._post_impl() except Exception as e: err_msg = f"{e.__class__.__name__} : {str(e)}" if err_msg != "KeyError : 'response'": diff --git a/tabpy/tabpy_server/management/state.py b/tabpy/tabpy_server/management/state.py index 26befa99..bbf50657 100644 --- a/tabpy/tabpy_server/management/state.py +++ b/tabpy/tabpy_server/management/state.py @@ -489,7 +489,7 @@ def get_access_control_allow_origin(self): """ _cors_origin = "" try: - logger.debug("Collecting Access-Control-Allow-Origin from " "state file...") + logger.debug("Collecting Access-Control-Allow-Origin from state file ...") _cors_origin = self._get_config_value( "Service Info", "Access-Control-Allow-Origin" ) @@ -591,22 +591,31 @@ def _get_config_items(self, section_name): def _get_config_value( self, section_name, option_name, optional=False, default_value=None ): + logger.log( + logging.DEBUG, + f"Loading option '{option_name}' from section [{section_name}]...") + if not self.config: - raise ValueError("State configuration not yet loaded.") + msg = "State configuration not yet loaded." + logging.log(msg) + raise ValueError(msg) + res = None if not option_name: - return self.config.options(section_name) - - if self.config.has_option(section_name, option_name): - return self.config.get(section_name, option_name) + res = self.config.options(section_name) + elif self.config.has_option(section_name, option_name): + res = self.config.get(section_name, option_name) elif optional: - return default_value + res = default_value else: raise ValueError( f"Cannot find option name {option_name} " f"under section {section_name}" ) + logger.log(logging.DEBUG, f"Returning value '{res}'") + return res + def _write_state(self, logger=logging.getLogger(__name__)): """ Write state (ConfigParser) to Consul diff --git a/tests/integration/test_custom_evaluate_timeout.py b/tests/integration/test_custom_evaluate_timeout.py index 2150c551..99bec14c 100644 --- a/tests/integration/test_custom_evaluate_timeout.py +++ b/tests/integration/test_custom_evaluate_timeout.py @@ -3,19 +3,19 @@ class TestCustomEvaluateTimeout(integ_test_base.IntegTestBase): def _get_evaluate_timeout(self) -> str: - return "5" + return "3" def test_custom_evaluate_timeout_with_script(self): # Uncomment the following line to preserve # test case output and other files (config, state, ect.) # in system temp folder. - # self.set_delete_temp_folder(False) + self.set_delete_temp_folder(False) payload = """ { "data": { "_arg1": 1 }, "script": - "import time\\nwhile True:\\n time.sleep(1)\\nreturn 1" + "import time\\ntime.sleep(100)\\nreturn 1" } """ headers = { @@ -29,10 +29,10 @@ def test_custom_evaluate_timeout_with_script(self): res = conn.getresponse() actual_error_message = res.read().decode("utf-8") + self.assertEqual(408, res.status) self.assertEqual( '{"message": ' - '"User defined script timed out. Timeout is set to 5.0 s.", ' + '"User defined script timed out. Timeout is set to 3.0 s.", ' '"info": {}}', actual_error_message, ) - self.assertEqual(408, res.status) diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index 40287371..3d524dd6 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -4,33 +4,15 @@ import tempfile from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.app.app import _init_asyncio_patch from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase -def _init_asyncio_patch(): - """ - Select compatible event loop for Tornado 5+. - As of Python 3.8, the default event loop on Windows is `proactor`, - however Tornado requires the old default "selector" event loop. - As Tornado has decided to leave this to users to set, MkDocs needs - to set it. See https://github.com/tornadoweb/tornado/issues/2608. - """ - if sys.platform.startswith("win") and sys.version_info >= (3, 8): - import asyncio - try: - from asyncio import WindowsSelectorEventLoopPolicy - except ImportError: - pass # Can't assign a policy which doesn't exist. - else: - if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy): - asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) - - class TestEndpointHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): - _init_asyncio_patch() + tabpy.tabpy_server.app.app._init_asyncio_patch() prefix = "__TestEndpointHandlerWithAuth_" # create password file cls.pwd_file = tempfile.NamedTemporaryFile( From 62bad450651ef2761b1758bf23ac53233bd2a1e4 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Fri, 10 Jan 2020 18:44:23 -0800 Subject: [PATCH 28/61] Code style improvements --- tabpy/tabpy_server/management/state.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tabpy/tabpy_server/management/state.py b/tabpy/tabpy_server/management/state.py index bbf50657..c36f7710 100644 --- a/tabpy/tabpy_server/management/state.py +++ b/tabpy/tabpy_server/management/state.py @@ -592,8 +592,8 @@ def _get_config_value( self, section_name, option_name, optional=False, default_value=None ): logger.log( - logging.DEBUG, - f"Loading option '{option_name}' from section [{section_name}]...") + logging.DEBUG, + f"Loading option '{option_name}' from section [{section_name}]...") if not self.config: msg = "State configuration not yet loaded." From 48c3e22186aef766e773b7fe9c807a2c8ef8c989 Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Fri, 10 Jan 2020 18:47:52 -0800 Subject: [PATCH 29/61] Code style improvements --- .github/workflows/pull_request.yml | 2 +- .github/workflows/push.yml | 2 +- tests/unit/server_tests/test_endpoint_handler.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 38d520d1..c3e8d5b9 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -1,4 +1,4 @@ -name: CI +name: Test Run on Pull Request on: [pull_request] diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 479d6957..d6c135be 100755 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -1,4 +1,4 @@ -name: CI +name: Test Run on Push on: [push] diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index 3d524dd6..a9393c42 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -12,7 +12,7 @@ class TestEndpointHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): - tabpy.tabpy_server.app.app._init_asyncio_patch() + _init_asyncio_patch() prefix = "__TestEndpointHandlerWithAuth_" # create password file cls.pwd_file = tempfile.NamedTemporaryFile( From 8d561d58dc0c21ee6758ca2903aeba7e7de8a4cd Mon Sep 17 00:00:00 2001 From: ogolovatyi Date: Fri, 14 Feb 2020 17:17:38 -0800 Subject: [PATCH 30/61] Add coverall workflow --- .github/workflows/coverage.yml | 35 ++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 .github/workflows/coverage.yml diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml new file mode 100755 index 00000000..7a6a037e --- /dev/null +++ b/.github/workflows/coverage.yml @@ -0,0 +1,35 @@ +name: Code coverage + +on: [push, pull_request] + +jobs: + build: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.7] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements.txt + pip install -r requirements_test.txt + pip install -r requirements_dev.txt + + - name: Test with pytest + run: | + pytest tests --cov=tabpy --cov-config=setup.cfg + coveralls + env: + COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} From 6ef4be77bf2fcb8c21dae70ad92c61406004f722 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Tue, 14 Apr 2020 15:19:22 -0700 Subject: [PATCH 31/61] Initial checkin, working tests. --- tabpy/tabpy_server/app/ConfigParameters.py | 1 + tabpy/tabpy_server/app/SettingsParameters.py | 1 + tabpy/tabpy_server/app/app.py | 1 + tabpy/tabpy_server/common/default.conf | 3 + .../handlers/service_info_handler.py | 13 +- .../server_tests/test_service_info_handler.py | 204 ++++++++++++++++++ 6 files changed, 221 insertions(+), 2 deletions(-) diff --git a/tabpy/tabpy_server/app/ConfigParameters.py b/tabpy/tabpy_server/app/ConfigParameters.py index 3feca9b7..351adfe5 100644 --- a/tabpy/tabpy_server/app/ConfigParameters.py +++ b/tabpy/tabpy_server/app/ConfigParameters.py @@ -15,3 +15,4 @@ class ConfigParameters: TABPY_STATIC_PATH = "TABPY_STATIC_PATH" TABPY_MAX_REQUEST_SIZE_MB = "TABPY_MAX_REQUEST_SIZE_MB" TABPY_EVALUATE_TIMEOUT = "TABPY_EVALUATE_TIMEOUT" + TABPY_AUTH_INFO = "TABPY_AUTH_INFO" diff --git a/tabpy/tabpy_server/app/SettingsParameters.py b/tabpy/tabpy_server/app/SettingsParameters.py index 45fb128a..218cd523 100755 --- a/tabpy/tabpy_server/app/SettingsParameters.py +++ b/tabpy/tabpy_server/app/SettingsParameters.py @@ -15,3 +15,4 @@ class SettingsParameters: StaticPath = "static_path" MaxRequestSizeInMb = "max_request_size_in_mb" EvaluateTimeout = "evaluate_timeout" + AuthInfo = "auth_info" diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 0ee807ad..69d396f8 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -271,6 +271,7 @@ def _parse_config(self, config_file): "false"), (SettingsParameters.MaxRequestSizeInMb, ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB, 100), + (SettingsParameters.AuthInfo, ConfigParameters.TABPY_AUTH_INFO, "false"), ] for setting, parameter, default_val in settings_parameters: diff --git a/tabpy/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf index ee02453d..9dab79ca 100755 --- a/tabpy/tabpy_server/common/default.conf +++ b/tabpy/tabpy_server/common/default.conf @@ -30,6 +30,9 @@ # The value should be a float representing the timeout time in seconds. # TABPY_EVALUATE_TIMEOUT = 30 +# Does the 'info' endpoint require authentication? +# TABPY_AUTH_INFO = false + [loggers] keys=root diff --git a/tabpy/tabpy_server/handlers/service_info_handler.py b/tabpy/tabpy_server/handlers/service_info_handler.py index 6b7060fb..e89045e7 100644 --- a/tabpy/tabpy_server/handlers/service_info_handler.py +++ b/tabpy/tabpy_server/handlers/service_info_handler.py @@ -8,9 +8,18 @@ def initialize(self, app): super(ServiceInfoHandler, self).initialize(app) def get(self): - # do not check for authentication - this method + # Optionally check for authentication - this method # is the only way for client to collect info about - # supported API versions and required features + # supported API versions and required features so auth is not checked + # by default. + # Some clients may wish to lock down the entire API which can be done through + # the configuration file. + + if self.settings[SettingsParameters.AuthInfo] == "true": + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + self._add_CORS_header() info = {} info["description"] = self.tabpy_state.get_description() diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index 9132c2fe..4e38af49 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -1,3 +1,4 @@ +import base64 import json import os from tabpy.tabpy_server.app.app import TabPyApp @@ -170,3 +171,206 @@ def test_tabpy_server_with_no_auth_expect_correct_info_response(self): self.assertTrue("features" in v1) features = v1["features"] self.assertDictEqual({}, features) + + + + +class TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword(AsyncHTTPTestCase): + @classmethod + def setUpClass(cls): + prefix = "__TestServiceInfoHandlerWithAuthWithSecureEndpoint_" + # create state.ini dir and file + cls.state_dir = tempfile.mkdtemp(prefix=prefix) + cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") + cls.state_file.write( + "[Service Info]\n" + "Name = TabPy Serve\n" + "Description = \n" + "Creation Time = 0\n" + "Access-Control-Allow-Origin = \n" + "Access-Control-Allow-Headers = \n" + "Access-Control-Allow-Methods = \n" + "\n" + "[Query Objects Service Versions]\n" + "\n" + "[Query Objects Docstrings]\n" + "\n" + "[Meta]\n" + "Revision Number = 1\n" + ) + cls.state_file.close() + + # create config file + cls.config_file = tempfile.NamedTemporaryFile( + prefix=prefix, suffix=".conf", delete=False + ) + cls.config_file.write( + bytes( + "[TabPy]\n" + "TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n" + "TABPY_AUTH_INFO = true\n", + "utf-8", + ) + ) + cls.config_file.close() + + @classmethod + def tearDownClass(cls): + os.remove(cls.state_file.name) + os.remove(cls.config_file.name) + os.rmdir(cls.state_dir) + + def get_app(self): + self.app = TabPyApp(self.config_file.name) + return self.app._create_tornado_web_app() + + def test_given_tabpy_server_with_auth_expect_correct_info_response(self): + + header = { + "Content-Type": "application/json", + "TabPy-Client": "Integration test for deploying models with auth", + "Authorization": "Basic " + + base64.b64encode("user1:P@ssw0rd".encode("utf-8")).decode("utf-8"), + } + + response = self.fetch("/info", headers=header) + self.assertEqual(response.code, 200) + actual_response = json.loads(response.body) + expected_response = _create_expected_info_response( + self.app.settings, self.app.tabpy_state + ) + + self.assertDictEqual(actual_response, expected_response) + self.assertTrue("versions" in actual_response) + versions = actual_response["versions"] + self.assertTrue("v1" in versions) + v1 = versions["v1"] + self.assertTrue("features" in v1) + features = v1["features"] + self.assertDictEqual( + {"authentication": {"methods": {"basic-auth": {}}, "required": True}}, + features, + ) + +class TestServiceInfoHandlerWithAuthWithSecureEndpoint(AsyncHTTPTestCase): + @classmethod + def setUpClass(cls): + prefix = "__TestServiceInfoHandlerWithAuthWithSecureEndpoint_" + # create password file + cls.pwd_file = tempfile.NamedTemporaryFile( + prefix=prefix, suffix=".txt", delete=False + ) + cls.pwd_file.write(b"username password") + cls.pwd_file.close() + + # create state.ini dir and file + cls.state_dir = tempfile.mkdtemp(prefix=prefix) + cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") + cls.state_file.write( + "[Service Info]\n" + "Name = TabPy Serve\n" + "Description = \n" + "Creation Time = 0\n" + "Access-Control-Allow-Origin = \n" + "Access-Control-Allow-Headers = \n" + "Access-Control-Allow-Methods = \n" + "\n" + "[Query Objects Service Versions]\n" + "\n" + "[Query Objects Docstrings]\n" + "\n" + "[Meta]\n" + "Revision Number = 1\n" + ) + cls.state_file.close() + + # create config file + cls.config_file = tempfile.NamedTemporaryFile( + prefix=prefix, suffix=".conf", delete=False + ) + cls.config_file.write( + bytes( + "[TabPy]\n" + f"TABPY_PWD_FILE = {cls.pwd_file.name}\n" + "TABPY_AUTH_INFO = true\n", + "utf-8", + ) + ) + cls.config_file.close() + + @classmethod + def tearDownClass(cls): + os.remove(cls.pwd_file.name) + os.remove(cls.state_file.name) + os.remove(cls.config_file.name) + os.rmdir(cls.state_dir) + + def get_app(self): + self.app = TabPyApp(self.config_file.name) + return self.app._create_tornado_web_app() + + def test_given_tabpy_server_with_auth_expect_correct_info_response(self): + response = self.fetch("/info") + self.assertEqual(response.code, 401) + +class TestServiceInfoHandlerWithoutAuthWithSecureEndpoint(AsyncHTTPTestCase): + @classmethod + def setUpClass(cls): + prefix = "__TestServiceInfoHandlerWithoutAuthWithSecureEndpoint_" + + # create state.ini dir and file + cls.state_dir = tempfile.mkdtemp(prefix=prefix) + with open(os.path.join(cls.state_dir, "state.ini"), "w+") as cls.state_file: + cls.state_file.write( + "[Service Info]\n" + "Name = TabPy Serve\n" + "Description = \n" + "Creation Time = 0\n" + "Access-Control-Allow-Origin = \n" + "Access-Control-Allow-Headers = \n" + "Access-Control-Allow-Methods = \n" + "\n" + "[Query Objects Service Versions]\n" + "\n" + "[Query Objects Docstrings]\n" + "\n" + "[Meta]\n" + "Revision Number = 1\n" + ) + cls.state_file.close() + + # create config file + cls.config_file = tempfile.NamedTemporaryFile( + prefix=prefix, suffix=".conf", delete=False, mode="w+" + ) + cls.config_file.write("[TabPy]\n" f"TABPY_STATE_PATH = {cls.state_dir}\n" + "TABPY_AUTH_INFO = true\n", + ) + cls.config_file.close() + + @classmethod + def tearDownClass(cls): + os.remove(cls.state_file.name) + os.remove(cls.config_file.name) + os.rmdir(cls.state_dir) + + def get_app(self): + self.app = TabPyApp(self.config_file.name) + return self.app._create_tornado_web_app() + + def test_tabpy_server_with_no_auth_expect_correct_info_response(self): + response = self.fetch("/info") + self.assertEqual(response.code, 200) + actual_response = json.loads(response.body) + expected_response = _create_expected_info_response( + self.app.settings, self.app.tabpy_state + ) + + self.assertDictEqual(actual_response, expected_response) + self.assertTrue("versions" in actual_response) + versions = actual_response["versions"] + self.assertTrue("v1" in versions) + v1 = versions["v1"] + self.assertTrue("features" in v1) + features = v1["features"] + self.assertDictEqual({}, features) From 3b10dcdc3419f8988ee69ba26f0852da2d921dce Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Tue, 14 Apr 2020 16:50:37 -0700 Subject: [PATCH 32/61] Made common base class for server info tests. --- .../server_tests/test_service_info_handler.py | 312 +++++------------- 1 file changed, 74 insertions(+), 238 deletions(-) diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index 4e38af49..300730bf 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -17,79 +17,80 @@ def _create_expected_info_response(settings, tabpy_state): "versions": settings["versions"], } - -class TestServiceInfoHandlerDefault(AsyncHTTPTestCase): +class BaseTestServiceInfoHandler(AsyncHTTPTestCase): def get_app(self): - self.app = TabPyApp() + if hasattr(self, 'config_file') and hasattr(self.config_file, 'name'): + self.app = TabPyApp(self.config_file.name) + else: + self.app = TabPyApp() return self.app._create_tornado_web_app() - def test_given_vanilla_tabpy_server_expect_correct_info_response(self): - response = self.fetch("/info") - self.assertEqual(response.code, 200) - actual_response = json.loads(response.body) - expected_response = _create_expected_info_response( - self.app.settings, self.app.tabpy_state - ) - - self.assertDictEqual(actual_response, expected_response) + @classmethod + def tearDownClass(cls): + os.remove(cls.state_file.name) + os.remove(cls.config_file.name) + os.rmdir(cls.state_dir) -class TestServiceInfoHandlerWithAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): - prefix = "__TestServiceInfoHandlerWithAuth_" - # create password file - cls.pwd_file = tempfile.NamedTemporaryFile( - prefix=prefix, suffix=".txt", delete=False - ) - cls.pwd_file.write(b"username password") - cls.pwd_file.close() - # create state.ini dir and file - cls.state_dir = tempfile.mkdtemp(prefix=prefix) - cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") - cls.state_file.write( - "[Service Info]\n" - "Name = TabPy Serve\n" - "Description = \n" - "Creation Time = 0\n" - "Access-Control-Allow-Origin = \n" - "Access-Control-Allow-Headers = \n" - "Access-Control-Allow-Methods = \n" - "\n" - "[Query Objects Service Versions]\n" - "\n" - "[Query Objects Docstrings]\n" - "\n" - "[Meta]\n" - "Revision Number = 1\n" - ) + cls.state_dir = tempfile.mkdtemp(prefix=cls.prefix) + with open(os.path.join(cls.state_dir, "state.ini"), "w+") as cls.state_file: + cls.state_file.write( + "[Service Info]\n" + "Name = TabPy Serve\n" + "Description = \n" + "Creation Time = 0\n" + "Access-Control-Allow-Origin = \n" + "Access-Control-Allow-Headers = \n" + "Access-Control-Allow-Methods = \n" + "\n" + "[Query Objects Service Versions]\n" + "\n" + "[Query Objects Docstrings]\n" + "\n" + "[Meta]\n" + "Revision Number = 1\n" + ) cls.state_file.close() # create config file cls.config_file = tempfile.NamedTemporaryFile( - prefix=prefix, suffix=".conf", delete=False - ) - cls.config_file.write( - bytes( - "[TabPy]\n" - f"TABPY_PWD_FILE = {cls.pwd_file.name}\n" - f"TABPY_STATE_PATH = {cls.state_dir}", - "utf-8", - ) + prefix=cls.prefix, suffix=".conf", delete=False, mode='w' ) + cls.config_file.write( "[TabPy]\n") + if hasattr(cls, 'tabpy_config'): + for k in cls.tabpy_config: + cls.config_file.write(k) cls.config_file.close() + +class TestServiceInfoHandlerDefault(BaseTestServiceInfoHandler): @classmethod - def tearDownClass(cls): - os.remove(cls.pwd_file.name) - os.remove(cls.state_file.name) - os.remove(cls.config_file.name) - os.rmdir(cls.state_dir) + def setUpClass(cls): + cls.prefix = "__TestServiceInfoHandlerWithAuth_" + cls.tabpy_config = ["TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] + super(TestServiceInfoHandlerDefault, cls).setUpClass() + + def test_given_vanilla_tabpy_server_expect_correct_info_response(self): + response = self.fetch("/info") + self.assertEqual(response.code, 200) + actual_response = json.loads(response.body) + expected_response = _create_expected_info_response( + self.app.settings, self.app.tabpy_state + ) + + self.assertDictEqual(actual_response, expected_response) + + +class TestServiceInfoHandlerWithAuth(BaseTestServiceInfoHandler): + @classmethod + def setUpClass(cls): + cls.prefix = "__TestServiceInfoHandlerWithAuth_" + cls.tabpy_config = ["TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] + super(TestServiceInfoHandlerWithAuth, cls).setUpClass() - def get_app(self): - self.app = TabPyApp(self.config_file.name) - return self.app._create_tornado_web_app() def test_given_tabpy_server_with_auth_expect_correct_info_response(self): response = self.fetch("/info") @@ -112,48 +113,11 @@ def test_given_tabpy_server_with_auth_expect_correct_info_response(self): ) -class TestServiceInfoHandlerWithoutAuth(AsyncHTTPTestCase): +class TestServiceInfoHandlerWithoutAuth(BaseTestServiceInfoHandler): @classmethod def setUpClass(cls): - prefix = "__TestServiceInfoHandlerWithoutAuth_" - - # create state.ini dir and file - cls.state_dir = tempfile.mkdtemp(prefix=prefix) - with open(os.path.join(cls.state_dir, "state.ini"), "w+") as cls.state_file: - cls.state_file.write( - "[Service Info]\n" - "Name = TabPy Serve\n" - "Description = \n" - "Creation Time = 0\n" - "Access-Control-Allow-Origin = \n" - "Access-Control-Allow-Headers = \n" - "Access-Control-Allow-Methods = \n" - "\n" - "[Query Objects Service Versions]\n" - "\n" - "[Query Objects Docstrings]\n" - "\n" - "[Meta]\n" - "Revision Number = 1\n" - ) - cls.state_file.close() - - # create config file - cls.config_file = tempfile.NamedTemporaryFile( - prefix=prefix, suffix=".conf", delete=False, mode="w+" - ) - cls.config_file.write("[TabPy]\n" f"TABPY_STATE_PATH = {cls.state_dir}") - cls.config_file.close() - - @classmethod - def tearDownClass(cls): - os.remove(cls.state_file.name) - os.remove(cls.config_file.name) - os.rmdir(cls.state_dir) - - def get_app(self): - self.app = TabPyApp(self.config_file.name) - return self.app._create_tornado_web_app() + cls.prefix = "__TestServiceInfoHandlerWithoutAuth_" + super(TestServiceInfoHandlerWithoutAuth, cls).setUpClass() def test_tabpy_server_with_no_auth_expect_correct_info_response(self): response = self.fetch("/info") @@ -175,54 +139,13 @@ def test_tabpy_server_with_no_auth_expect_correct_info_response(self): -class TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword(AsyncHTTPTestCase): +class TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword(BaseTestServiceInfoHandler): @classmethod def setUpClass(cls): - prefix = "__TestServiceInfoHandlerWithAuthWithSecureEndpoint_" - # create state.ini dir and file - cls.state_dir = tempfile.mkdtemp(prefix=prefix) - cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") - cls.state_file.write( - "[Service Info]\n" - "Name = TabPy Serve\n" - "Description = \n" - "Creation Time = 0\n" - "Access-Control-Allow-Origin = \n" - "Access-Control-Allow-Headers = \n" - "Access-Control-Allow-Methods = \n" - "\n" - "[Query Objects Service Versions]\n" - "\n" - "[Query Objects Docstrings]\n" - "\n" - "[Meta]\n" - "Revision Number = 1\n" - ) - cls.state_file.close() - - # create config file - cls.config_file = tempfile.NamedTemporaryFile( - prefix=prefix, suffix=".conf", delete=False - ) - cls.config_file.write( - bytes( - "[TabPy]\n" - "TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n" - "TABPY_AUTH_INFO = true\n", - "utf-8", - ) - ) - cls.config_file.close() - - @classmethod - def tearDownClass(cls): - os.remove(cls.state_file.name) - os.remove(cls.config_file.name) - os.rmdir(cls.state_dir) - - def get_app(self): - self.app = TabPyApp(self.config_file.name) - return self.app._create_tornado_web_app() + cls.prefix = "__TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword_" + cls.tabpy_config = ["TABPY_AUTH_INFO = true\n", + "TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] + super(TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword, cls).setUpClass() def test_given_tabpy_server_with_auth_expect_correct_info_response(self): @@ -252,111 +175,24 @@ def test_given_tabpy_server_with_auth_expect_correct_info_response(self): features, ) -class TestServiceInfoHandlerWithAuthWithSecureEndpoint(AsyncHTTPTestCase): +class TestServiceInfoHandlerWithAuthWithSecureEndpoint(BaseTestServiceInfoHandler): @classmethod def setUpClass(cls): - prefix = "__TestServiceInfoHandlerWithAuthWithSecureEndpoint_" - # create password file - cls.pwd_file = tempfile.NamedTemporaryFile( - prefix=prefix, suffix=".txt", delete=False - ) - cls.pwd_file.write(b"username password") - cls.pwd_file.close() - - # create state.ini dir and file - cls.state_dir = tempfile.mkdtemp(prefix=prefix) - cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") - cls.state_file.write( - "[Service Info]\n" - "Name = TabPy Serve\n" - "Description = \n" - "Creation Time = 0\n" - "Access-Control-Allow-Origin = \n" - "Access-Control-Allow-Headers = \n" - "Access-Control-Allow-Methods = \n" - "\n" - "[Query Objects Service Versions]\n" - "\n" - "[Query Objects Docstrings]\n" - "\n" - "[Meta]\n" - "Revision Number = 1\n" - ) - cls.state_file.close() - - # create config file - cls.config_file = tempfile.NamedTemporaryFile( - prefix=prefix, suffix=".conf", delete=False - ) - cls.config_file.write( - bytes( - "[TabPy]\n" - f"TABPY_PWD_FILE = {cls.pwd_file.name}\n" - "TABPY_AUTH_INFO = true\n", - "utf-8", - ) - ) - cls.config_file.close() - - @classmethod - def tearDownClass(cls): - os.remove(cls.pwd_file.name) - os.remove(cls.state_file.name) - os.remove(cls.config_file.name) - os.rmdir(cls.state_dir) - - def get_app(self): - self.app = TabPyApp(self.config_file.name) - return self.app._create_tornado_web_app() + cls.prefix = "__TestServiceInfoHandlerWithAuthWithSecureEndpoint_" + cls.tabpy_config = ["TABPY_AUTH_INFO = true\n", + "TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] + super(TestServiceInfoHandlerWithAuthWithSecureEndpoint, cls).setUpClass() def test_given_tabpy_server_with_auth_expect_correct_info_response(self): response = self.fetch("/info") self.assertEqual(response.code, 401) -class TestServiceInfoHandlerWithoutAuthWithSecureEndpoint(AsyncHTTPTestCase): +class TestServiceInfoHandlerWithoutAuthWithSecureEndpoint(BaseTestServiceInfoHandler): @classmethod def setUpClass(cls): - prefix = "__TestServiceInfoHandlerWithoutAuthWithSecureEndpoint_" - - # create state.ini dir and file - cls.state_dir = tempfile.mkdtemp(prefix=prefix) - with open(os.path.join(cls.state_dir, "state.ini"), "w+") as cls.state_file: - cls.state_file.write( - "[Service Info]\n" - "Name = TabPy Serve\n" - "Description = \n" - "Creation Time = 0\n" - "Access-Control-Allow-Origin = \n" - "Access-Control-Allow-Headers = \n" - "Access-Control-Allow-Methods = \n" - "\n" - "[Query Objects Service Versions]\n" - "\n" - "[Query Objects Docstrings]\n" - "\n" - "[Meta]\n" - "Revision Number = 1\n" - ) - cls.state_file.close() - - # create config file - cls.config_file = tempfile.NamedTemporaryFile( - prefix=prefix, suffix=".conf", delete=False, mode="w+" - ) - cls.config_file.write("[TabPy]\n" f"TABPY_STATE_PATH = {cls.state_dir}\n" - "TABPY_AUTH_INFO = true\n", - ) - cls.config_file.close() - - @classmethod - def tearDownClass(cls): - os.remove(cls.state_file.name) - os.remove(cls.config_file.name) - os.rmdir(cls.state_dir) - - def get_app(self): - self.app = TabPyApp(self.config_file.name) - return self.app._create_tornado_web_app() + cls.prefix = "__TestServiceInfoHandlerWithoutAuthWithSecureEndpoint_" + cls.tabpy_config = ["TABPY_AUTH_INFO = true\n"] + super(TestServiceInfoHandlerWithoutAuthWithSecureEndpoint, cls).setUpClass() def test_tabpy_server_with_no_auth_expect_correct_info_response(self): response = self.fetch("/info") From c3719ef8b69cb6f1a47bdcd75f00655b7171de0b Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Tue, 14 Apr 2020 17:00:20 -0700 Subject: [PATCH 33/61] pep8 checks. --- .../server_tests/test_service_info_handler.py | 20 +++++++++---------- 1 file changed, 9 insertions(+), 11 deletions(-) diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index 300730bf..82688e8d 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -17,6 +17,7 @@ def _create_expected_info_response(settings, tabpy_state): "versions": settings["versions"], } + class BaseTestServiceInfoHandler(AsyncHTTPTestCase): def get_app(self): if hasattr(self, 'config_file') and hasattr(self.config_file, 'name'): @@ -31,7 +32,6 @@ def tearDownClass(cls): os.remove(cls.config_file.name) os.rmdir(cls.state_dir) - @classmethod def setUpClass(cls): # create state.ini dir and file @@ -59,10 +59,10 @@ def setUpClass(cls): cls.config_file = tempfile.NamedTemporaryFile( prefix=cls.prefix, suffix=".conf", delete=False, mode='w' ) - cls.config_file.write( "[TabPy]\n") + cls.config_file.write("[TabPy]\n") if hasattr(cls, 'tabpy_config'): - for k in cls.tabpy_config: - cls.config_file.write(k) + for k in cls.tabpy_config: + cls.config_file.write(k) cls.config_file.close() @@ -91,7 +91,6 @@ def setUpClass(cls): cls.tabpy_config = ["TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] super(TestServiceInfoHandlerWithAuth, cls).setUpClass() - def test_given_tabpy_server_with_auth_expect_correct_info_response(self): response = self.fetch("/info") self.assertEqual(response.code, 200) @@ -137,23 +136,20 @@ def test_tabpy_server_with_no_auth_expect_correct_info_response(self): self.assertDictEqual({}, features) - - class TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword(BaseTestServiceInfoHandler): @classmethod def setUpClass(cls): cls.prefix = "__TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword_" cls.tabpy_config = ["TABPY_AUTH_INFO = true\n", - "TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] + "TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] super(TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword, cls).setUpClass() def test_given_tabpy_server_with_auth_expect_correct_info_response(self): - header = { "Content-Type": "application/json", "TabPy-Client": "Integration test for deploying models with auth", - "Authorization": "Basic " - + base64.b64encode("user1:P@ssw0rd".encode("utf-8")).decode("utf-8"), + "Authorization": "Basic " + + base64.b64encode("user1:P@ssw0rd".encode("utf-8")).decode("utf-8"), } response = self.fetch("/info", headers=header) @@ -175,6 +171,7 @@ def test_given_tabpy_server_with_auth_expect_correct_info_response(self): features, ) + class TestServiceInfoHandlerWithAuthWithSecureEndpoint(BaseTestServiceInfoHandler): @classmethod def setUpClass(cls): @@ -187,6 +184,7 @@ def test_given_tabpy_server_with_auth_expect_correct_info_response(self): response = self.fetch("/info") self.assertEqual(response.code, 401) + class TestServiceInfoHandlerWithoutAuthWithSecureEndpoint(BaseTestServiceInfoHandler): @classmethod def setUpClass(cls): From 937ed274a95e45d151577e27545cf9cb25620123 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Tue, 14 Apr 2020 17:17:04 -0700 Subject: [PATCH 34/61] Added documentation for TABPY_AUTH_INFO --- docs/server-config.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/server-config.md b/docs/server-config.md index 03f57ef8..f6c29903 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -71,6 +71,10 @@ at [`logging.config` documentation page](https://docs.python.org/3.6/library/log makes TabPy require credentials with HTTP(S) requests. More details about authentication can be found in [Authentication](#authentication) section. Default value - not set. +- `TABPY_AUTH_INFO` - Determines whether authorization is required when + accessing the `info` API. More details about + authentication can be found in [Authentication](#authentication) + section. Default value - False. - `TABPY_TRANSFER_PROTOCOL` - transfer protocol. Default value - `http`. If set to `https` two additional parameters have to be specified: `TABPY_CERTIFICATE_FILE` and `TABPY_KEY_FILE`. @@ -109,6 +113,7 @@ settings._ # For how to configure TabPy authentication read # docs/server-config.md. # TABPY_PWD_FILE = /path/to/password/file.txt +# TABPY_AUTH_INFO = true # To set up secure TabPy uncomment and modify the following lines. # Note only PEM-encoded x509 certificates are supported. @@ -245,6 +250,10 @@ will be generated and displayed in the command line. To delete an account open password file in any text editor and delete the line with the user name. +### Endpoint Security +All endpoints except `info` require authentication if it is enabled for the server. + `info` can be secured using the `TABPY_AUTH_INFO` setting. + ## Logging Logging for TabPy is implemented with Python's standard logger and can be configured From ab50457e1ba9e8cf27f3e3e03f2745e993c7ab86 Mon Sep 17 00:00:00 2001 From: Logan Riggs Date: Tue, 14 Apr 2020 17:30:45 -0700 Subject: [PATCH 35/61] Fix spacing. --- docs/server-config.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/server-config.md b/docs/server-config.md index f6c29903..282c07ae 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -71,7 +71,7 @@ at [`logging.config` documentation page](https://docs.python.org/3.6/library/log makes TabPy require credentials with HTTP(S) requests. More details about authentication can be found in [Authentication](#authentication) section. Default value - not set. -- `TABPY_AUTH_INFO` - Determines whether authorization is required when +- `TABPY_AUTH_INFO` - Determines whether authorization is required when accessing the `info` API. More details about authentication can be found in [Authentication](#authentication) section. Default value - False. @@ -251,6 +251,7 @@ To delete an account open password file in any text editor and delete the line with the user name. ### Endpoint Security + All endpoints except `info` require authentication if it is enabled for the server. `info` can be secured using the `TABPY_AUTH_INFO` setting. From 870f26550f1eb78a6e8dd4ae2f36f9148c4643dc Mon Sep 17 00:00:00 2001 From: lriggs Date: Wed, 15 Apr 2020 12:48:21 -0700 Subject: [PATCH 36/61] Refactor config parsing to allow custom parsers. (#412) * Refactor config parsing to allow custom parsers. * Fix pep8 * Update version and changelog. * Changed default for tabpy_auth_info to a boolean. --- CHANGELOG | 11 +++++ tabpy/VERSION | 2 +- tabpy/tabpy_server/app/app.py | 49 ++++++++----------- .../handlers/service_info_handler.py | 2 +- tests/unit/server_tests/test_config.py | 46 ++++++++++++++++- 5 files changed, 78 insertions(+), 32 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index c57d48fc..49ce5644 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,16 @@ # Changelog +## v1.1.0 + +### Improvements + +- Added ability to require authorization for the /info API method. + This is set with TABPY_AUTH_INFO. +- Improved config parsing flexibility. Previously the + TABPY_EVALUATE_TIMEOUT setting would be set to a default if + tabpy couldn't parse the value. Now it will throw an exception + at startup. + ## v1.0.0 ### Improvements diff --git a/tabpy/VERSION b/tabpy/VERSION index 3eefcb9d..9084fa2f 100755 --- a/tabpy/VERSION +++ b/tabpy/VERSION @@ -1 +1 @@ -1.0.0 +1.1.0 diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 69d396f8..b15550d8 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -183,7 +183,7 @@ def try_exit(self): return application - def _set_parameter(self, parser, settings_key, config_key, default_val): + def _set_parameter(self, parser, settings_key, config_key, default_val, parse_function): key_is_set = False if ( @@ -191,7 +191,9 @@ def _set_parameter(self, parser, settings_key, config_key, default_val): and parser.has_section("TabPy") and parser.has_option("TabPy", config_key) ): - self.settings[settings_key] = parser.get("TabPy", config_key) + if parse_function is None: + parse_function = parser.get + self.settings[settings_key] = parse_function("TabPy", config_key) key_is_set = True logger.debug( f"Parameter {settings_key} set to " @@ -252,41 +254,32 @@ def _parse_config(self, config_file): ) settings_parameters = [ - (SettingsParameters.Port, ConfigParameters.TABPY_PORT, 9004), - (SettingsParameters.ServerVersion, None, __version__), - (SettingsParameters.EvaluateTimeout, ConfigParameters.TABPY_EVALUATE_TIMEOUT, 30), + (SettingsParameters.Port, ConfigParameters.TABPY_PORT, 9004, None), + (SettingsParameters.ServerVersion, None, __version__, None), + (SettingsParameters.EvaluateTimeout, ConfigParameters.TABPY_EVALUATE_TIMEOUT, + 30, parser.getfloat), (SettingsParameters.UploadDir, ConfigParameters.TABPY_QUERY_OBJECT_PATH, - os.path.join(pkg_path, "tmp", "query_objects")), + os.path.join(pkg_path, "tmp", "query_objects"), None), (SettingsParameters.TransferProtocol, ConfigParameters.TABPY_TRANSFER_PROTOCOL, - "http"), + "http", None), (SettingsParameters.CertificateFile, ConfigParameters.TABPY_CERTIFICATE_FILE, - None), - (SettingsParameters.KeyFile, ConfigParameters.TABPY_KEY_FILE, None), + None, None), + (SettingsParameters.KeyFile, ConfigParameters.TABPY_KEY_FILE, None, None), (SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, - os.path.join(pkg_path, "tabpy_server")), + os.path.join(pkg_path, "tabpy_server"), None), (SettingsParameters.StaticPath, ConfigParameters.TABPY_STATIC_PATH, - os.path.join(pkg_path, "tabpy_server", "static")), - (ConfigParameters.TABPY_PWD_FILE, ConfigParameters.TABPY_PWD_FILE, None), + os.path.join(pkg_path, "tabpy_server", "static"), None), + (ConfigParameters.TABPY_PWD_FILE, ConfigParameters.TABPY_PWD_FILE, None, None), (SettingsParameters.LogRequestContext, ConfigParameters.TABPY_LOG_DETAILS, - "false"), + "false", None), (SettingsParameters.MaxRequestSizeInMb, ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB, - 100), - (SettingsParameters.AuthInfo, ConfigParameters.TABPY_AUTH_INFO, "false"), + 100, None), + (SettingsParameters.AuthInfo, ConfigParameters.TABPY_AUTH_INFO, False, + parser.getboolean), ] - for setting, parameter, default_val in settings_parameters: - self._set_parameter(parser, setting, parameter, default_val) - - try: - self.settings[SettingsParameters.EvaluateTimeout] = float( - self.settings[SettingsParameters.EvaluateTimeout] - ) - except ValueError: - logger.warning( - "Evaluate timeout must be a float type. Defaulting " - "to evaluate timeout of 30 seconds." - ) - self.settings[SettingsParameters.EvaluateTimeout] = 30 + for setting, parameter, default_val, parse_function in settings_parameters: + self._set_parameter(parser, setting, parameter, default_val, parse_function) if not os.path.exists(self.settings[SettingsParameters.UploadDir]): os.makedirs(self.settings[SettingsParameters.UploadDir]) diff --git a/tabpy/tabpy_server/handlers/service_info_handler.py b/tabpy/tabpy_server/handlers/service_info_handler.py index e89045e7..4fd329bb 100644 --- a/tabpy/tabpy_server/handlers/service_info_handler.py +++ b/tabpy/tabpy_server/handlers/service_info_handler.py @@ -15,7 +15,7 @@ def get(self): # Some clients may wish to lock down the entire API which can be done through # the configuration file. - if self.settings[SettingsParameters.AuthInfo] == "true": + if self.settings[SettingsParameters.AuthInfo]: if self.should_fail_with_not_authorized(): self.fail_with_not_authorized() return diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 84c6cd5f..e7776723 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -116,6 +116,48 @@ def test_config_file_present( self.assertEqual(app.settings["log_request_context"], False) self.assertEqual(app.settings["evaluate_timeout"], 30) + @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) + @patch("tabpy.tabpy_server.app.app._get_state_from_file") + @patch("tabpy.tabpy_server.app.app.TabPyState") + def test_info_auth_valid( + self, mock_state, mock_get_state_from_file, mock_path_exists + ): + self.assertTrue(self.config_file is not None) + config_file = self.config_file + config_file.write("[TabPy]\n" "TABPY_AUTH_INFO = True".encode()) + config_file.close() + + app = TabPyApp(self.config_file.name) + self.assertEqual(app.settings["auth_info"], True) + + @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) + @patch("tabpy.tabpy_server.app.app._get_state_from_file") + @patch("tabpy.tabpy_server.app.app.TabPyState") + def test_info_auth_camelcase_valid( + self, mock_state, mock_get_state_from_file, mock_path_exists + ): + self.assertTrue(self.config_file is not None) + config_file = self.config_file + config_file.write("[TabPy]\n" "TABPY_AUTH_INFO = trUE".encode()) + config_file.close() + + app = TabPyApp(self.config_file.name) + self.assertEqual(app.settings["auth_info"], True) + + @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) + @patch("tabpy.tabpy_server.app.app._get_state_from_file") + @patch("tabpy.tabpy_server.app.app.TabPyState") + def test_info_auth_false_valid( + self, mock_state, mock_get_state_from_file, mock_path_exists + ): + self.assertTrue(self.config_file is not None) + config_file = self.config_file + config_file.write("[TabPy]\n" "TABPY_AUTH_INFO = no".encode()) + config_file.close() + + app = TabPyApp(self.config_file.name) + self.assertEqual(app.settings["auth_info"], False) + @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) @patch("tabpy.tabpy_server.app.app._get_state_from_file") @patch("tabpy.tabpy_server.app.app.TabPyState") @@ -143,8 +185,8 @@ def test_custom_evaluate_timeout_invalid( ) config_file.close() - app = TabPyApp(self.config_file.name) - self.assertEqual(app.settings["evaluate_timeout"], 30.0) + with self.assertRaises(ValueError) as err: + TabPyApp(self.config_file.name) @patch("tabpy.tabpy_server.app.app.os") @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) From 3ae024dcf9d74247df86f4466b7a80c195c68d29 Mon Sep 17 00:00:00 2001 From: lriggs Date: Tue, 21 Apr 2020 12:01:49 -0700 Subject: [PATCH 37/61] Secure info (#414) * Update README.md * Doc update (#402) * Fixed broken link * Linked to install doc. * Dev fix spelling (#408) * Add spelling fix workflow * Refactor config parsing to allow custom parsers. * Fix pep8 * Update version and changelog. * Changed default for tabpy_auth_info to a boolean. * Remove configuration for securing the info API and make it secure by default. * Revert "Merge branch 'master' into secureInfo" This reverts commit 368856193acae0a40327fbb2560c7d7448fa5999, reversing changes made to a4acc65bbc58cbad4ca20725d3d0d70127596603. * Removed auth config from uni test. Co-authored-by: nmannheimer Co-authored-by: Oleksandr Golovatyi --- CHANGELOG | 5 +- docs/server-config.md | 8 +- tabpy/tabpy_server/app/ConfigParameters.py | 1 - tabpy/tabpy_server/app/SettingsParameters.py | 1 - tabpy/tabpy_server/app/app.py | 2 - .../handlers/service_info_handler.py | 14 +--- tests/unit/server_tests/test_config.py | 42 ---------- .../server_tests/test_service_info_handler.py | 82 ++++--------------- 8 files changed, 21 insertions(+), 134 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 49ce5644..1988386d 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -4,8 +4,9 @@ ### Improvements -- Added ability to require authorization for the /info API method. - This is set with TABPY_AUTH_INFO. +- Authorization is now required for the /info API method. + This method did not check authentication previously. This change is + backwards compatible with Tableau clients. - Improved config parsing flexibility. Previously the TABPY_EVALUATE_TIMEOUT setting would be set to a default if tabpy couldn't parse the value. Now it will throw an exception diff --git a/docs/server-config.md b/docs/server-config.md index 282c07ae..d759635e 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -71,10 +71,6 @@ at [`logging.config` documentation page](https://docs.python.org/3.6/library/log makes TabPy require credentials with HTTP(S) requests. More details about authentication can be found in [Authentication](#authentication) section. Default value - not set. -- `TABPY_AUTH_INFO` - Determines whether authorization is required when - accessing the `info` API. More details about - authentication can be found in [Authentication](#authentication) - section. Default value - False. - `TABPY_TRANSFER_PROTOCOL` - transfer protocol. Default value - `http`. If set to `https` two additional parameters have to be specified: `TABPY_CERTIFICATE_FILE` and `TABPY_KEY_FILE`. @@ -113,7 +109,6 @@ settings._ # For how to configure TabPy authentication read # docs/server-config.md. # TABPY_PWD_FILE = /path/to/password/file.txt -# TABPY_AUTH_INFO = true # To set up secure TabPy uncomment and modify the following lines. # Note only PEM-encoded x509 certificates are supported. @@ -252,8 +247,7 @@ line with the user name. ### Endpoint Security -All endpoints except `info` require authentication if it is enabled for the server. - `info` can be secured using the `TABPY_AUTH_INFO` setting. +All endpoints require authentication if it is enabled for the server. ## Logging diff --git a/tabpy/tabpy_server/app/ConfigParameters.py b/tabpy/tabpy_server/app/ConfigParameters.py index 351adfe5..3feca9b7 100644 --- a/tabpy/tabpy_server/app/ConfigParameters.py +++ b/tabpy/tabpy_server/app/ConfigParameters.py @@ -15,4 +15,3 @@ class ConfigParameters: TABPY_STATIC_PATH = "TABPY_STATIC_PATH" TABPY_MAX_REQUEST_SIZE_MB = "TABPY_MAX_REQUEST_SIZE_MB" TABPY_EVALUATE_TIMEOUT = "TABPY_EVALUATE_TIMEOUT" - TABPY_AUTH_INFO = "TABPY_AUTH_INFO" diff --git a/tabpy/tabpy_server/app/SettingsParameters.py b/tabpy/tabpy_server/app/SettingsParameters.py index 218cd523..45fb128a 100755 --- a/tabpy/tabpy_server/app/SettingsParameters.py +++ b/tabpy/tabpy_server/app/SettingsParameters.py @@ -15,4 +15,3 @@ class SettingsParameters: StaticPath = "static_path" MaxRequestSizeInMb = "max_request_size_in_mb" EvaluateTimeout = "evaluate_timeout" - AuthInfo = "auth_info" diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index b15550d8..563b1c65 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -274,8 +274,6 @@ def _parse_config(self, config_file): "false", None), (SettingsParameters.MaxRequestSizeInMb, ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB, 100, None), - (SettingsParameters.AuthInfo, ConfigParameters.TABPY_AUTH_INFO, False, - parser.getboolean), ] for setting, parameter, default_val, parse_function in settings_parameters: diff --git a/tabpy/tabpy_server/handlers/service_info_handler.py b/tabpy/tabpy_server/handlers/service_info_handler.py index 4fd329bb..51152a98 100644 --- a/tabpy/tabpy_server/handlers/service_info_handler.py +++ b/tabpy/tabpy_server/handlers/service_info_handler.py @@ -8,17 +8,9 @@ def initialize(self, app): super(ServiceInfoHandler, self).initialize(app) def get(self): - # Optionally check for authentication - this method - # is the only way for client to collect info about - # supported API versions and required features so auth is not checked - # by default. - # Some clients may wish to lock down the entire API which can be done through - # the configuration file. - - if self.settings[SettingsParameters.AuthInfo]: - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return self._add_CORS_header() info = {} diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index e7776723..41e5d814 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -116,48 +116,6 @@ def test_config_file_present( self.assertEqual(app.settings["log_request_context"], False) self.assertEqual(app.settings["evaluate_timeout"], 30) - @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) - @patch("tabpy.tabpy_server.app.app._get_state_from_file") - @patch("tabpy.tabpy_server.app.app.TabPyState") - def test_info_auth_valid( - self, mock_state, mock_get_state_from_file, mock_path_exists - ): - self.assertTrue(self.config_file is not None) - config_file = self.config_file - config_file.write("[TabPy]\n" "TABPY_AUTH_INFO = True".encode()) - config_file.close() - - app = TabPyApp(self.config_file.name) - self.assertEqual(app.settings["auth_info"], True) - - @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) - @patch("tabpy.tabpy_server.app.app._get_state_from_file") - @patch("tabpy.tabpy_server.app.app.TabPyState") - def test_info_auth_camelcase_valid( - self, mock_state, mock_get_state_from_file, mock_path_exists - ): - self.assertTrue(self.config_file is not None) - config_file = self.config_file - config_file.write("[TabPy]\n" "TABPY_AUTH_INFO = trUE".encode()) - config_file.close() - - app = TabPyApp(self.config_file.name) - self.assertEqual(app.settings["auth_info"], True) - - @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) - @patch("tabpy.tabpy_server.app.app._get_state_from_file") - @patch("tabpy.tabpy_server.app.app.TabPyState") - def test_info_auth_false_valid( - self, mock_state, mock_get_state_from_file, mock_path_exists - ): - self.assertTrue(self.config_file is not None) - config_file = self.config_file - config_file.write("[TabPy]\n" "TABPY_AUTH_INFO = no".encode()) - config_file.close() - - app = TabPyApp(self.config_file.name) - self.assertEqual(app.settings["auth_info"], False) - @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) @patch("tabpy.tabpy_server.app.app._get_state_from_file") @patch("tabpy.tabpy_server.app.app.TabPyState") diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index 82688e8d..4eb82f65 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -66,24 +66,6 @@ def setUpClass(cls): cls.config_file.close() -class TestServiceInfoHandlerDefault(BaseTestServiceInfoHandler): - @classmethod - def setUpClass(cls): - cls.prefix = "__TestServiceInfoHandlerWithAuth_" - cls.tabpy_config = ["TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] - super(TestServiceInfoHandlerDefault, cls).setUpClass() - - def test_given_vanilla_tabpy_server_expect_correct_info_response(self): - response = self.fetch("/info") - self.assertEqual(response.code, 200) - actual_response = json.loads(response.body) - expected_response = _create_expected_info_response( - self.app.settings, self.app.tabpy_state - ) - - self.assertDictEqual(actual_response, expected_response) - - class TestServiceInfoHandlerWithAuth(BaseTestServiceInfoHandler): @classmethod def setUpClass(cls): @@ -91,8 +73,19 @@ def setUpClass(cls): cls.tabpy_config = ["TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] super(TestServiceInfoHandlerWithAuth, cls).setUpClass() - def test_given_tabpy_server_with_auth_expect_correct_info_response(self): + def test_given_tabpy_server_with_auth_expect_error_info_response(self): response = self.fetch("/info") + self.assertEqual(response.code, 401) + + def test_given_tabpy_server_with_auth_expect_correct_info_response(self): + header = { + "Content-Type": "application/json", + "TabPy-Client": "Integration test for deploying models with auth", + "Authorization": "Basic " + + base64.b64encode("user1:P@ssw0rd".encode("utf-8")).decode("utf-8"), + } + + response = self.fetch("/info", headers=header) self.assertEqual(response.code, 200) actual_response = json.loads(response.body) expected_response = _create_expected_info_response( @@ -135,16 +128,7 @@ def test_tabpy_server_with_no_auth_expect_correct_info_response(self): features = v1["features"] self.assertDictEqual({}, features) - -class TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword(BaseTestServiceInfoHandler): - @classmethod - def setUpClass(cls): - cls.prefix = "__TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword_" - cls.tabpy_config = ["TABPY_AUTH_INFO = true\n", - "TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] - super(TestServiceInfoHandlerWithAuthWithSecureEndpointAndPassword, cls).setUpClass() - - def test_given_tabpy_server_with_auth_expect_correct_info_response(self): + def test_given_tabpy_server_with_no_auth_and_password_expect_correct_info_response(self): header = { "Content-Type": "application/json", "TabPy-Client": "Integration test for deploying models with auth", @@ -167,44 +151,6 @@ def test_given_tabpy_server_with_auth_expect_correct_info_response(self): self.assertTrue("features" in v1) features = v1["features"] self.assertDictEqual( - {"authentication": {"methods": {"basic-auth": {}}, "required": True}}, + {}, features, ) - - -class TestServiceInfoHandlerWithAuthWithSecureEndpoint(BaseTestServiceInfoHandler): - @classmethod - def setUpClass(cls): - cls.prefix = "__TestServiceInfoHandlerWithAuthWithSecureEndpoint_" - cls.tabpy_config = ["TABPY_AUTH_INFO = true\n", - "TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] - super(TestServiceInfoHandlerWithAuthWithSecureEndpoint, cls).setUpClass() - - def test_given_tabpy_server_with_auth_expect_correct_info_response(self): - response = self.fetch("/info") - self.assertEqual(response.code, 401) - - -class TestServiceInfoHandlerWithoutAuthWithSecureEndpoint(BaseTestServiceInfoHandler): - @classmethod - def setUpClass(cls): - cls.prefix = "__TestServiceInfoHandlerWithoutAuthWithSecureEndpoint_" - cls.tabpy_config = ["TABPY_AUTH_INFO = true\n"] - super(TestServiceInfoHandlerWithoutAuthWithSecureEndpoint, cls).setUpClass() - - def test_tabpy_server_with_no_auth_expect_correct_info_response(self): - response = self.fetch("/info") - self.assertEqual(response.code, 200) - actual_response = json.loads(response.body) - expected_response = _create_expected_info_response( - self.app.settings, self.app.tabpy_state - ) - - self.assertDictEqual(actual_response, expected_response) - self.assertTrue("versions" in actual_response) - versions = actual_response["versions"] - self.assertTrue("v1" in versions) - v1 = versions["v1"] - self.assertTrue("features" in v1) - features = v1["features"] - self.assertDictEqual({}, features) From a7bdfcb9c1c40060990f84c85d7db9961b7abf21 Mon Sep 17 00:00:00 2001 From: lriggs Date: Thu, 23 Apr 2020 16:44:45 -0700 Subject: [PATCH 38/61] Secure info (#417) * Update README.md * Doc update (#402) * Fixed broken link * Linked to install doc. * Dev fix spelling (#408) * Add spelling fix workflow * Refactor config parsing to allow custom parsers. * Fix pep8 * Update version and changelog. * Changed default for tabpy_auth_info to a boolean. * Remove configuration for securing the info API and make it secure by default. * Revert "Merge branch 'master' into secureInfo" This reverts commit 368856193acae0a40327fbb2560c7d7448fa5999, reversing changes made to a4acc65bbc58cbad4ca20725d3d0d70127596603. * Removed auth config from uni test. * Removed example for removed setting. * Remove unused example config. Co-authored-by: nmannheimer Co-authored-by: Oleksandr Golovatyi --- tabpy/tabpy_server/common/default.conf | 3 --- 1 file changed, 3 deletions(-) diff --git a/tabpy/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf index 9dab79ca..ee02453d 100755 --- a/tabpy/tabpy_server/common/default.conf +++ b/tabpy/tabpy_server/common/default.conf @@ -30,9 +30,6 @@ # The value should be a float representing the timeout time in seconds. # TABPY_EVALUATE_TIMEOUT = 30 -# Does the 'info' endpoint require authentication? -# TABPY_AUTH_INFO = false - [loggers] keys=root From 66c2c8f15755e51d3eaeb90160fbbb95fab0ae34 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Wed, 20 May 2020 09:35:08 -0700 Subject: [PATCH 39/61] Clean up API documentation (#420) * Update README.md * Doc update (#402) * Fixed broken link * Linked to install doc. * Dev fix spelling (#408) * Add spelling fix workflow * Update LICENSE update to Tableau Software LLC * v 1.1.0: Secure /info with auth (#415) - Authorization is now required for the /info API method. This method did not check authentication previously. This change is backwards compatible with Tableau clients. - Improved config parsing flexibility. Previously the TABPY_EVALUATE_TIMEOUT setting would be set to a default if tabpy couldn't parse the value. Now it will throw an exception at startup. * Clean up API documentation * Clean up API documentation * Clean up API documentation Co-authored-by: nmannheimer Co-authored-by: lriggs Co-authored-by: Olek Golovatyi --- .github/workflows/pull_fix_spelling.yml | 22 ++ LICENSE | 2 +- README.md | 4 +- docs/FAQ.md | 26 --- docs/TableauConfiguration.md | 2 +- docs/api-v1.md | 280 ------------------------ docs/server-config.md | 2 +- docs/server-install.md | 2 +- docs/server-rest.md | 202 +++++++++++------ docs/tabpy-tools.md | 51 +++-- 10 files changed, 188 insertions(+), 405 deletions(-) create mode 100644 .github/workflows/pull_fix_spelling.yml delete mode 100644 docs/FAQ.md delete mode 100755 docs/api-v1.md diff --git a/.github/workflows/pull_fix_spelling.yml b/.github/workflows/pull_fix_spelling.yml new file mode 100644 index 00000000..c13e520b --- /dev/null +++ b/.github/workflows/pull_fix_spelling.yml @@ -0,0 +1,22 @@ +name: Fix spelling on push + +on: [push] + +jobs: + build: + name: ${{ matrix.python-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + + strategy: + matrix: + python-version: [3.7] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v2 + - uses: sobolevn/misspell-fixer-action@master + - uses: peter-evans/create-pull-request@v2.4.4 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: 'Fixes by misspell-fixer' + title: 'Typos fix by misspell-fixer' diff --git a/LICENSE b/LICENSE index 09f14442..b9702de2 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ MIT License -Copyright (c) 2016 Tableau +Copyright (c) 2016 Tableau Software, LLC Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/README.md b/README.md index 7c288cb2..8df8df84 100755 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ [![PyPI version](https://badge.fury.io/py/tabpy.svg)](https://pypi.python.org/pypi/tabpy/) ![Release](https://img.shields.io/github/release/tableau/TabPy.svg) -TabPy (the Tableau Python Server) is an Analytices Extension implementation which +TabPy (the Tableau Python Server) is an Analytics Extension implementation which expands Tableau's capabilities by allowing users to execute Python scripts and saved functions via Tableau's table calculations. @@ -30,7 +30,7 @@ Consider reading TabPy documentation in the following order: Troubleshooting: -* [FAQ for configuration, startup and other issues](docs/FAQ.md) +* [TabPy Wiki](https://github.com/tableau/TabPy/wiki) More technical topics: diff --git a/docs/FAQ.md b/docs/FAQ.md deleted file mode 100644 index 916f410d..00000000 --- a/docs/FAQ.md +++ /dev/null @@ -1,26 +0,0 @@ -# TabPy Frequently Asked Questions - - - - - -- [Startup Issues](#startup-issues) - * [AttributeError: module 'tornado.web' has no attribute 'asynchronous'](#attributeerror-module-tornadoweb-has-no-attribute-asynchronous) - * [NotImplementedError](#notimplementederror) - - - - - -## Startup Issues - -### AttributeError: module 'tornado.web' has no attribute 'asynchronous' - -TabPy uses Tornado 5.1.1. To it to your Python environment run -`pip install tornado==5.1.1` and then try to start TabPy again. - -### NotImplementedError - -Check your Python version (`python -V` command) - TabPy doesn't work with -Python 3.8. It is recommended to use Python 3.6 or 3.7 for now. -The root cause for the failure - [Tornado issue](https://github.com/tornadoweb/tornado/issues/2608). diff --git a/docs/TableauConfiguration.md b/docs/TableauConfiguration.md index 770dc9c5..6fb5fadd 100755 --- a/docs/TableauConfiguration.md +++ b/docs/TableauConfiguration.md @@ -15,7 +15,7 @@ ## Configuration -Once you have a [TabPy instance](server-startup.md) set up you can easily +Once you have a [TabPy instance](server-install.md) set up you can easily configure Tableau to use this service for evaluating Python code. ### Tableau Desktop diff --git a/docs/api-v1.md b/docs/api-v1.md deleted file mode 100755 index 6841b2b0..00000000 --- a/docs/api-v1.md +++ /dev/null @@ -1,280 +0,0 @@ -# TabPy API v1 - - - - - -- [Authentication](#authentication) -- [http:get:: /status](#httpget-status) -- [http:get:: /endpoints](#httpget-endpoints) -- [http:get:: /endpoints/:endpoint](#httpget-endpointsendpoint) -- [http:post:: /evaluate](#httppost-evaluate) -- [http:post:: /query/:endpoint](#httppost-queryendpoint) - - - - - -## Authentication - -When authentication is enabled for v1 API [`/info` call](server-rest.md#get-info), -the response contains authentication feature parameters, e.g.: - - ```json - { - "description": "", - "creation_time": "0", - "state_path": "e:\\dev\\TabPy\\tabpy-server\\tabpy_server", - "server_version": "0.4.1", - "name": "TabPy Server", - "versions": { - "v1": { - "features": { - "authentication": { - "required": true, - "methods": { - "basic-auth": {} - } - } - } - } - } - } - ``` - -v1 authentication specific features (see the example above): - - - -Property | Description ---- | --- -`required` | Authentication is never optional for a client to use if it is in the features list. -`methods` | List of supported authentication methods with their properties. -`methods.basic-auth` | TabPy requires basic access authentication. See [TabPy Server Configuration Instructions](server-config.md#authentication) for how to configure authentication. - - - -## http:get:: /status - -Gets runtime status of deployed endpoints. If no endpoints are deployed in -the server, the returned data is an empty JSON object. - -Example request: - -```HTTP -GET /status HTTP/1.1 -Host: localhost:9004 -Accept: application/json -``` - -Example response: - -```HTTP -HTTP/1.1 200 OK -Content-Type: application/json - -{"clustering": { - "status": "LoadSuccessful", - "last_error": null, - "version": 1, - "type": "model"}, - "add": { - "status": "LoadSuccessful", - "last_error": null, - "version": 1, - "type": "model"} -} -``` - -Using curl: - -```bash -curl -X GET http://localhost:9004/status -``` - -## http:get:: /endpoints - -Gets a list of deployed endpoints and their static information. If no -endpoints are deployed in the server, the returned data is an empty JSON object. - -Example request: - -```HTTP -GET /endpoints HTTP/1.1 -Host: localhost:9004 -Accept: application/json -``` - -Example response: - -```HTTP -HTTP/1.1 200 OK -Content-Type: application/json - -{"clustering": - {"description": "", - "docstring": "-- no docstring found in query function --", - "creation_time": 1469511182, - "version": 1, - "dependencies": [], - "last_modified_time": 1469511182, - "type": "model", - "target": null}, -"add": { - "description": "", - "docstring": "-- no docstring found in query function --", - "creation_time": 1469505967, - "version": 1, - "dependencies": [], - "last_modified_time": 1469505967, - "type": "model", - "target": null} -} -``` - -Using curl: - -```bash -curl -X GET http://localhost:9004/endpoints -``` - -## http:get:: /endpoints/:endpoint - -Gets the description of a specific deployed endpoint. The endpoint must first -be deployed in the server (see the [TabPy Tools documentation](tabpy-tools.md)). - -Example request: - -```HTTP -GET /endpoints/add HTTP/1.1 -Host: localhost:9004 -Accept: application/json -``` - -Example response: - -```HTTP -HTTP/1.1 200 OK -Content-Type: application/json - -{"description": "", "docstring": "-- no docstring found in query function --", - "creation_time": 1469505967, "version": 1, "dependencies": [], - "last_modified_time": 1469505967, "type": "model", "target": null} -``` - -Using curl: - -```bash -curl -X GET http://localhost:9004/endpoints/add -``` - -## http:post:: /evaluate - -Executes a block of Python code, replacing named parameters with their provided -values. - -The expected POST body is a JSON dictionary with two elements: - -- A key `data` with a value that contains the parameter values passed to the - code. These values are key-value pairs, following a specific convention for - key names (`_arg1`, `_arg2`, etc.). -- A key `script` with a value that contains the Python code (one or more lines). - Any references to the parameter names will be replaced by their values - according to `data`. - -Example request: - -```HTTP -POST /evaluate HTTP/1.1 -Host: localhost:9004 -Accept: application/json - -{"data": {"_arg1": 1, "_arg2": 2}, "script": "return _arg1+_arg2"} -``` - -Example response: - -```HTTP -HTTP/1.1 200 OK -Content-Type: application/json - -3 -``` - -Using curl: - -```bash -curl -X POST http://localhost:9004/evaluate \ --d '{"data": {"_arg1": 1, "_arg2": 2}, "script": "return _arg1 + _arg2"}' -``` - -It is possible to call a deployed function from within the code block through -the predefined function `tabpy.query`. This function works like the client -library's `query` method, and returns the corresponding data structure. The -function must first be deployed as an endpoint in the server (for more details -see the [TabPy Tools documentation](tabpy-tools.md)). - -The following example calls the endpoint `clustering` as it was deployed in the -section [deploy-function](tabpy-tools.md#deploying-a-function): - -```HTTP -POST /evaluate HTTP/1.1 -Host: example.com -Accept: application/json - -{ "data": - { "_arg1": [6.35, 6.40, 6.65, 8.60, 8.90, 9.00, 9.10], - "_arg2": [1.95, 1.95, 2.05, 3.05, 3.05, 3.10, 3.15] - }, - "script": "return tabpy.query('clustering', x=_arg1, y=_arg2)"} -``` - -The next example shows how to call `evaluate` from a terminal using curl. This -code queries the method `add` that was deployed in the section -[deploy-function](tabpy-tools.md#deploying-a-function): - -```bash -curl -X POST http://localhost:9004/evaluate \ --d '{"data": {"_arg1":1, "_arg2":2}, - "script": "return tabpy.query(\"add\", x=_arg1, y=_arg2)[\"response\"]"}' -``` - -## http:post:: /query/:endpoint - -Executes a function at the specified endpoint. The function must first be -deployed (see the [TabPy Tools documentation](tabpy-tools.md)). - -This interface expects a JSON body with a `data` key, specifying the values -for the function, according to its original definition. In the example below, -the function `clustering` was defined with a signature of two parameters `x` -and `y`, expecting arrays of numbers. - -Example request: - -```HTTP -POST /query/clustering HTTP/1.1 -Host: localhost:9004 -Accept: application/json - -{"data": { - "x": [6.35, 6.40, 6.65, 8.60, 8.90, 9.00, 9.10], - "y": [1.95, 1.95, 2.05, 3.05, 3.05, 3.10, 3.15]}} -``` - -Example response: - -```HTTP -HTTP/1.1 200 OK -Content-Type: application/json - -{"model": "clustering", "version": 1, "response": [0, 0, 0, 1, 1, 1, 1], - "uuid": "46d3df0e-acca-4560-88f1-67c5aedeb1c4"} -``` - -Using curl: - -```bash -curl -X GET http://localhost:9004/query/clustering -d \ -'{"data": {"x": [6.35, 6.40, 6.65, 8.60, 8.90, 9.00, 9.10], - "y": [1.95, 1.95, 2.05, 3.05, 3.05, 3.10, 3.15]}}' -``` diff --git a/docs/server-config.md b/docs/server-config.md index d759635e..28fa0deb 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -31,7 +31,7 @@ Configuration parameters can be updated with: 1. Adding environment variables - set the environment variable as required by your Operating System. When creating environment variables, use the same name for your environment variable as specified in the config file. -2. Specifying a parameter in a config file (enviroment variable value overwrites +2. Specifying a parameter in a config file (environment variable value overwrites configuration setting). Configuration file with custom settings is specified as a command line parameter: diff --git a/docs/server-install.md b/docs/server-install.md index e91aa4dc..4eabe0d2 100755 --- a/docs/server-install.md +++ b/docs/server-install.md @@ -44,6 +44,6 @@ and specify it in command line: tabpy --config=path/to/my/config/file.conf ``` -It is highly recommended to use Python virtual enviroment for running TabPy. +It is highly recommended to use Python virtual environment for running TabPy. Check the [Running TabPy in Python Virtual Environment](tabpy-virtualenv.md) page for more details. diff --git a/docs/server-rest.md b/docs/server-rest.md index 3bae2952..73980425 100755 --- a/docs/server-rest.md +++ b/docs/server-rest.md @@ -7,105 +7,173 @@ Python code and query deployed methods. -- [GET /info](#get-info) - * [URL](#url) - * [Method](#method) - * [URL parameters](#url-parameters) - * [Data Parameters](#data-parameters) - * [Response](#response) -- [API versions](#api-versions) +- [Authentication, /info and /evaluate](#authentication-info-and-evaluate) +- [http:get:: /status](#httpget-status) +- [http:get:: /endpoints](#httpget-endpoints) +- [http:get:: /endpoints/:endpoint](#httpget-endpointsendpoint) +- [http:post:: /query/:endpoint](#httppost-queryendpoint) -## GET /info +## Authentication, /info and /evaluate -Get static information about the server. The method doesn't require any -authentication and returns supported API versions which the client can use -together with optional and required features. +Analytics Extensions API v1 is documented at +[https://tableau.github.io/analytics-extensions-api/docs/ae_api_ref.html](https://tableau.github.io/analytics-extensions-api/docs/ae_api_ref.html). -### URL +The following documentation is for methods not currently used by Tableau. + +## http:get:: /status + +Gets runtime status of deployed endpoints. If no endpoints are deployed in +the server, the returned data is an empty JSON object. + +Example request: ```HTTP -/info +GET /status HTTP/1.1 +Host: localhost:9004 +Accept: application/json ``` -### Method +Example response: ```HTTP -GET +HTTP/1.1 200 OK +Content-Type: application/json + +{"clustering": { + "status": "LoadSuccessful", + "last_error": null, + "version": 1, + "type": "model"}, + "add": { + "status": "LoadSuccessful", + "last_error": null, + "version": 1, + "type": "model"} +} +``` + +Using curl: + +```bash +curl -X GET http://localhost:9004/status ``` -### URL parameters +## http:get:: /endpoints + +Gets a list of deployed endpoints and their static information. If no +endpoints are deployed in the server, the returned data is an empty JSON object. -None. +Example request: -### Data Parameters +```HTTP +GET /endpoints HTTP/1.1 +Host: localhost:9004 +Accept: application/json +``` -None. +Example response: -### Response +```HTTP +HTTP/1.1 200 OK +Content-Type: application/json + +{"clustering": + {"description": "", + "docstring": "-- no docstring found in query function --", + "creation_time": 1469511182, + "version": 1, + "dependencies": [], + "last_modified_time": 1469511182, + "type": "model", + "target": null}, +"add": { + "description": "", + "docstring": "-- no docstring found in query function --", + "creation_time": 1469505967, + "version": 1, + "dependencies": [], + "last_modified_time": 1469505967, + "type": "model", + "target": null} +} +``` -For a successful call: +Using curl: -- Status: 200 -- Content: +```bash +curl -X GET http://localhost:9004/endpoints +``` - ```json - { - "description": "", - "creation_time": "0", - "state_path": "e:\\dev\\TabPy\\tabpy-server\\tabpy_server", - "server_version": "0.4.1", - "name": "TabPy Server", - "versions": { - "v1": { - "features": { - "authentication": { - "required": true, - "methods": { - "basic-auth": {} - } - } - } - } - } - } - ``` +## http:get:: /endpoints/:endpoint -Response fields: +Gets the description of a specific deployed endpoint. The endpoint must first +be deployed in the server (see the [TabPy Tools documentation](tabpy-tools.md)). - +Example request: -Property | Description ---- | --- -`description` | String that is hardcoded in the `state.ini` file and can be edited there. -`creation_time` | Creation time in seconds since 1970-01-01, hardcoded in the `state.ini` file, where it can be edited. -`state_path` | State file path of the server (the value of the TABPY_STATE_PATH at the time the server was started). -`server_version` | TabPy Server version tag. Clients can use this information for compatibility checks. -`name` | TabPy server instance name. Can be edited in `state.ini` file. -`version` | Collection of API versions supported by the server. Each entry in the collection is an API version which has a corresponding list of properties. -`version.`*``* | Set of properties for an API version. -`version.`*`.features`* | Set of an API's available features. -`version.`*`.features.`* | Set of a feature's properties. For specific details for a particular property meaning of a feature, check the documentation for the specific API version. -`version.`*`.features..required`* | If true the feature is required to be used by client. +```HTTP +GET /endpoints/add HTTP/1.1 +Host: localhost:9004 +Accept: application/json +``` - +Example response: -See [TabPy Configuration](#tabpy-configuration) section for more information -on modifying the settings. +```HTTP +HTTP/1.1 200 OK +Content-Type: application/json -- **Examples** +{"description": "", "docstring": "-- no docstring found in query function --", + "creation_time": 1469505967, "version": 1, "dependencies": [], + "last_modified_time": 1469505967, "type": "model", "target": null} +``` -Calling the method with curl: +Using curl: ```bash -curl -X GET http://localhost:9004/info +curl -X GET http://localhost:9004/endpoints/add +``` + +## http:post:: /query/:endpoint + +Executes a function at the specified endpoint. The function must first be +deployed (see the [TabPy Tools documentation](tabpy-tools.md)). + +This interface expects a JSON body with a `data` key, specifying the values +for the function, according to its original definition. In the example below, +the function `clustering` was defined with a signature of two parameters `x` +and `y`, expecting arrays of numbers. + +Example request: + +```HTTP +POST /query/clustering HTTP/1.1 +Host: localhost:9004 +Accept: application/json + +{"data": { + "x": [6.35, 6.40, 6.65, 8.60, 8.90, 9.00, 9.10], + "y": [1.95, 1.95, 2.05, 3.05, 3.05, 3.10, 3.15]}} ``` -## API versions +Example response: + +```HTTP +HTTP/1.1 200 OK +Content-Type: application/json + +{"model": "clustering", "version": 1, "response": [0, 0, 0, 1, 1, 1, 1], + "uuid": "46d3df0e-acca-4560-88f1-67c5aedeb1c4"} +``` -TabPy supports the following API versions: +Using curl: -- v1 - see details at [api-v1.md](api-v1.md). +```bash +curl -X GET http://localhost:9004/query/clustering -d \ +'{"data": {"x": [6.35, 6.40, 6.65, 8.60, 8.90, 9.00, 9.10], + "y": [1.95, 1.95, 2.05, 3.05, 3.05, 3.10, 3.15]}}' +``` diff --git a/docs/tabpy-tools.md b/docs/tabpy-tools.md index 70f3b9d0..73717fd4 100755 --- a/docs/tabpy-tools.md +++ b/docs/tabpy-tools.md @@ -290,32 +290,31 @@ tabpy.query(‘ttest’, _arg1, _arg2)[‘response’] and is capable of performing two types of t-tests: - -1\. [A t-test for the means of two independent samples with equal variance](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.ttest_ind.html) -This is a two-sided t test with the null hypothesis being that the mean of -sample1 is equal to the mean of sample2. -_arg1 (list of numeric values): a list of independent observations -_arg2 (list of numeric values): a list of independent observations equal to -the length of _arg1 - -Alternatively, your data may not be split into separate measures. If that is -the case you can pass the following fields to ttest, - -_arg1 (list of numeric values): a list of independent observations -_arg2 (list of categorical variables with cardinality two): a binary factor -that maps each observation in _arg1 to either sample1 or sample2 (this list -should be equal to the length of _arg1) - -2\. [A t-test for the mean of one group](https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.ttest_1samp.html) -_arg1 (list of numeric values): a list of independent observations -_arg2 (a numeric value): the known population mean -A two-sided t test with the null hypothesis being that the mean of a sample of -independent observations is equal to the given population mean. - -The function returns a two-tailed [p-value](https://en.wikipedia.org/wiki/P-value) -(between 0 and 1). Depending on your [significance level](https://en.wikipedia.org/wiki/Statistical_significance) -you may reject or fail to reject the null hypothesis. - +1. [A t-test for the means of two independent samples with equal variance](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.ttest_ind.html) + This is a two-sided t test with the null hypothesis being that the mean of + sample1 is equal to the mean of sample2: + + - `_arg1` (list of numeric values): a list of independent observations. + - `_arg2` (list of numeric values): a list of independent observations equal to + the length of `_arg1`. + + Alternatively, your data may not be split into separate measures. If that is + the case you can pass the following fields to ttest: + + - `_arg1` (list of numeric values): a list of independent observations + - `_arg2` (list of categorical variables with cardinality two): a binary factor + that maps each observation in `_arg1` to either sample1 or sample2 (this list + should be equal to the length of `_arg1`). + +2. [A t-test for the mean of one group](https://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.stats.ttest_1samp.html): + - `_arg1` (list of numeric values): a list of independent observations. + - `_arg2` (a numeric value): the known population mean + A two-sided t test with the null hypothesis being that the mean of a sample + of independent observations is equal to the given population mean. + + The function returns a two-tailed [p-value](https://en.wikipedia.org/wiki/P-value) + (between 0 and 1). Depending on your [significance level](https://en.wikipedia.org/wiki/Statistical_significance) + you may reject or fail to reject the null hypothesis. ### ANOVA From c43d9b192eb44c86f6718a84d3269cfe030b6b2b Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Mon, 20 Jul 2020 15:18:16 -0700 Subject: [PATCH 40/61] Code improvements (#431) * Rename tabpy_server and tabpy_tools to server and tools (breaking change) * fix flake8 warnings * Clean up code to reduce number of conditions --- .gitignore | 7 +- .vscode/settings.json | 5 +- CHANGELOG | 11 ++ CONTRIBUTING.md | 2 +- MANIFEST.in | 8 +- docs/security.md | 2 +- docs/server-config.md | 10 +- docs/tabpy-tools.md | 14 +- setup.py | 6 +- tabpy/VERSION | 2 +- tabpy/models/deploy_models.py | 23 +-- tabpy/models/utils/setup_utils.py | 15 +- tabpy/{tabpy_server => server}/__init__.py | 0 .../app/ConfigParameters.py | 0 .../app/SettingsParameters.py | 0 .../{tabpy_server => server}/app/__init__.py | 0 tabpy/{tabpy_server => server}/app/app.py | 30 ++-- tabpy/{tabpy_server => server}/app/util.py | 0 .../common/__init__.py | 0 .../common/default.conf | 4 +- .../common/endpoint_file_mgr.py | 0 .../common/messages.py | 0 tabpy/{tabpy_server => server}/common/util.py | 0 tabpy/server/handlers/__init__.py | 13 ++ .../handlers/base_handler.py | 15 +- .../handlers/endpoint_handler.py | 10 +- .../handlers/endpoints_handler.py | 4 +- .../handlers/evaluation_plane_handler.py | 13 +- .../handlers/main_handler.py | 2 +- .../handlers/management_handler.py | 58 ++++---- .../handlers/query_plane_handler.py | 6 +- .../handlers/service_info_handler.py | 4 +- .../handlers/status_handler.py | 2 +- .../handlers/upload_destination_handler.py | 4 +- .../{tabpy_server => server}/handlers/util.py | 64 ++++----- .../management/__init__.py | 0 .../management/state.py | 133 ++++++++++-------- .../management/util.py | 4 +- .../{tabpy_server => server}/psws/__init__.py | 0 .../psws/callbacks.py | 12 +- .../psws/python_service.py | 6 +- .../state.ini.template | 30 ++-- .../static/index.html | 0 .../static/tableau.png | Bin tabpy/tabpy.py | 2 +- tabpy/tabpy_server/handlers/__init__.py | 13 -- tabpy/{tabpy_tools => tools}/__init__.py | 0 tabpy/{tabpy_tools => tools}/client.py | 16 +-- .../custom_query_object.py | 0 tabpy/{tabpy_tools => tools}/query_object.py | 0 tabpy/{tabpy_tools => tools}/rest.py | 0 tabpy/{tabpy_tools => tools}/rest_client.py | 0 tabpy/{tabpy_tools => tools}/schema.py | 2 +- tabpy/utils/tabpy_user.py | 4 +- .../resources/deploy_and_evaluate_model.conf | 4 +- tests/unit/server_tests/test_config.py | 68 ++++----- .../test_endpoint_file_manager.py | 2 +- .../server_tests/test_endpoint_handler.py | 6 +- .../server_tests/test_endpoints_handler.py | 4 +- .../test_evaluation_plane_handler.py | 4 +- tests/unit/server_tests/test_pwd_file.py | 2 +- .../server_tests/test_service_info_handler.py | 12 +- tests/unit/tools_tests/test_client.py | 4 +- tests/unit/tools_tests/test_rest.py | 2 +- tests/unit/tools_tests/test_rest_object.py | 2 +- tests/unit/tools_tests/test_schema.py | 2 +- 66 files changed, 329 insertions(+), 339 deletions(-) rename tabpy/{tabpy_server => server}/__init__.py (100%) rename tabpy/{tabpy_server => server}/app/ConfigParameters.py (100%) rename tabpy/{tabpy_server => server}/app/SettingsParameters.py (100%) mode change 100755 => 100644 rename tabpy/{tabpy_server => server}/app/__init__.py (100%) rename tabpy/{tabpy_server => server}/app/app.py (94%) rename tabpy/{tabpy_server => server}/app/util.py (100%) rename tabpy/{tabpy_server => server}/common/__init__.py (100%) rename tabpy/{tabpy_server => server}/common/default.conf (94%) mode change 100755 => 100644 rename tabpy/{tabpy_server => server}/common/endpoint_file_mgr.py (100%) rename tabpy/{tabpy_server => server}/common/messages.py (100%) rename tabpy/{tabpy_server => server}/common/util.py (100%) create mode 100644 tabpy/server/handlers/__init__.py rename tabpy/{tabpy_server => server}/handlers/base_handler.py (96%) rename tabpy/{tabpy_server => server}/handlers/endpoint_handler.py (93%) rename tabpy/{tabpy_server => server}/handlers/endpoints_handler.py (95%) rename tabpy/{tabpy_server => server}/handlers/evaluation_plane_handler.py (91%) rename tabpy/{tabpy_server => server}/handlers/main_handler.py (70%) rename tabpy/{tabpy_server => server}/handlers/management_handler.py (71%) rename tabpy/{tabpy_server => server}/handlers/query_plane_handler.py (97%) rename tabpy/{tabpy_server => server}/handlers/service_info_handler.py (85%) rename tabpy/{tabpy_server => server}/handlers/status_handler.py (93%) rename tabpy/{tabpy_server => server}/handlers/upload_destination_handler.py (79%) rename tabpy/{tabpy_server => server}/handlers/util.py (96%) mode change 100755 => 100644 rename tabpy/{tabpy_server => server}/management/__init__.py (100%) rename tabpy/{tabpy_server => server}/management/state.py (85%) rename tabpy/{tabpy_server => server}/management/util.py (90%) rename tabpy/{tabpy_server => server}/psws/__init__.py (100%) rename tabpy/{tabpy_server => server}/psws/callbacks.py (94%) rename tabpy/{tabpy_server => server}/psws/python_service.py (98%) rename tabpy/{tabpy_server => server}/state.ini.template (94%) mode change 100755 => 100644 rename tabpy/{tabpy_server => server}/static/index.html (100%) rename tabpy/{tabpy_server => server}/static/tableau.png (100%) delete mode 100644 tabpy/tabpy_server/handlers/__init__.py rename tabpy/{tabpy_tools => tools}/__init__.py (100%) mode change 100755 => 100644 rename tabpy/{tabpy_tools => tools}/client.py (96%) rename tabpy/{tabpy_tools => tools}/custom_query_object.py (100%) mode change 100755 => 100644 rename tabpy/{tabpy_tools => tools}/query_object.py (100%) mode change 100755 => 100644 rename tabpy/{tabpy_tools => tools}/rest.py (100%) mode change 100755 => 100644 rename tabpy/{tabpy_tools => tools}/rest_client.py (100%) mode change 100755 => 100644 rename tabpy/{tabpy_tools => tools}/schema.py (98%) mode change 100755 => 100644 diff --git a/.gitignore b/.gitignore index 621c6072..1ebc2208 100644 --- a/.gitignore +++ b/.gitignore @@ -117,9 +117,9 @@ package-lock.json .idea/ # TabPy server artifacts -tabpy/tabpy_server/state.ini -tabpy/tabpy_server/query_objects -tabpy/tabpy_server/staging +tabpy/server/state.ini +tabpy/server/query_objects +tabpy/server/staging # VS Code *.code-workspace @@ -128,3 +128,4 @@ tabpy/tabpy_server/staging # etc setup.bat *~ +tabpy_log.log.1 diff --git a/.vscode/settings.json b/.vscode/settings.json index e27fae0a..2b4fa02e 100755 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -9,7 +9,7 @@ "**/*.pyc": true }, "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": false, + "python.linting.flake8Enabled": true, "python.linting.enabled": true, "python.testing.autoTestDiscoverOnSaveEnabled": true, "python.testing.pytestArgs": [ @@ -18,5 +18,6 @@ "python.testing.unittestEnabled": false, "python.testing.nosetestsEnabled": false, "python.testing.pytestEnabled": true, - "python.linting.pycodestyleEnabled": true + "python.linting.pycodestyleEnabled": false, + "python.pythonPath": "C:\\Users\\ogolovatyi\\Anaconda3\\envs\\Python 37\\python.exe" } \ No newline at end of file diff --git a/CHANGELOG b/CHANGELOG index 1988386d..8714766d 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,16 @@ # Changelog +## v2.0.0 + +### Improvements + +- Minor code cleanup. + +### Breaking changes + +- tabpy_server and tabpy_tools are renamed to server and tools. + + ## v1.1.0 ### Improvements diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc87d783..5baaef84 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,7 @@ and run it locally. These are prerequisites for an environment required for a contributor to be able to work on TabPy changes: -- Python 3.6 or 3.7: +- Python 3.6, 3.7 or 3.8: - To see which version of Python you have installed, run `python --version`. - git - Node.js for npm packages - install from . diff --git a/MANIFEST.in b/MANIFEST.in index 2d93e579..55289a56 100755 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,10 @@ exclude \ - tabpy/tabpy_server/state.ini + tabpy/server/state.ini include \ CHANGELOG \ LICENSE \ tabpy/VERSION \ - tabpy/tabpy_server/state.ini.template \ - tabpy/tabpy_server/static/* \ - tabpy/tabpy_server/common/default.conf + tabpy/server/state.ini.template \ + tabpy/server/static/* \ + tabpy/server/common/default.conf diff --git a/docs/security.md b/docs/security.md index 5902b2d3..5f8a3c2a 100755 --- a/docs/security.md +++ b/docs/security.md @@ -6,7 +6,7 @@ you may want to consider the following as you use TabPy: - The REST server and Python execution share the same Python session, meaning that HTTP requests and user scripts are evaluated in the same addressable memory and processor threads. -- The tabpy_tools client does not perform client-side validation of the +- The tabpy.tools client does not perform client-side validation of the SSL certificate on TabPy Server. - Python scripts can contain code which can harm security on the server where the TabPy is running. For example, Python scripts can: diff --git a/docs/server-config.md b/docs/server-config.md index 28fa0deb..543304da 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -62,10 +62,10 @@ at [`logging.config` documentation page](https://docs.python.org/3.6/library/log [TabPy Tools documentation](tabpy-tools.md) for details. Default value - `/tmp/query_objects`. - `TABPY_STATE_PATH` - state folder location (absolute path) for Tornado web - server. Default value - `tabpy/tabpy_server` subfolder in TabPy package + server. Default value - `tabpy/server` subfolder in TabPy package folder. - `TABPY_STATIC_PATH` - absolute path for location of static files (index.html - page) for TabPy instance. Default value - `tabpy/tabpy_server/static` + page) for TabPy instance. Default value - `tabpy/server/static` subfolder in TabPy package folder. - `TABPY_PWD_FILE` - absolute path to password file. Setting up this parameter makes TabPy require credentials with HTTP(S) requests. More details about @@ -101,10 +101,10 @@ settings._ [TabPy] # TABPY_QUERY_OBJECT_PATH = /tmp/query_objects # TABPY_PORT = 9004 -# TABPY_STATE_PATH = /tabpy/tabpy_server +# TABPY_STATE_PATH = /tabpy/server # Where static pages live -# TABPY_STATIC_PATH = /tabpy/tabpy_server/static +# TABPY_STATIC_PATH = /tabpy/server/static # For how to configure TabPy authentication read # docs/server-config.md. @@ -256,7 +256,7 @@ as explained in Python documentation at [Logging Configuration page](https://docs.python.org/3.6/library/logging.config.html). A default config provided with TabPy is at -[`tabpy-server/tabpy_server/common/default.conf`](tabpy-server/tabpy_server/common/default.conf) +[`tabpy-server/server/common/default.conf`](tabpy-server/server/common/default.conf) and has a configuration for console and file loggers. Changing the config file allows the user to modify the log level, format of the logged messages and add or remove loggers. diff --git a/docs/tabpy-tools.md b/docs/tabpy-tools.md index 73717fd4..a7143156 100755 --- a/docs/tabpy-tools.md +++ b/docs/tabpy-tools.md @@ -30,7 +30,7 @@ to specify the service location for all subsequent operations: ```python -from tabpy.tabpy_tools.client import Client +from tabpy.tools.client import Client client = Client('http://localhost:9004/') @@ -351,7 +351,7 @@ method provided in this tools package: ```python -from tabpy_tools.schema import generate_schema +from tabpy.tools.schema import generate_schema schema = generate_schema( input={'x': 3, 'y': 2}, @@ -368,7 +368,7 @@ To describe more complex input, like arrays, you would use the following syntax: ```python -from tabpy_tools.schema import generate_schema +from tabpy.tools.schema import generate_schema schema = generate_schema( input={'x': [6.35, 6.40, 6.65, 8.60], @@ -413,10 +413,10 @@ Response: ```json { - 'model': 'clustering', - 'response': [0, 0, 0, 1, 1, 1, 1], - 'uuid': '1ca01e46-733c-4a77-b3da-3ded84dff4cd', - 'version': 2 + "model": "clustering", + "response": [0, 0, 0, 1, 1, 1, 1], + "uuid": "1ca01e46-733c-4a77-b3da-3ded84dff4cd", + "version": 2 } ``` diff --git a/setup.py b/setup.py index d4fd80bd..11828310 100755 --- a/setup.py +++ b/setup.py @@ -54,9 +54,9 @@ def read(fname): package_data={ "tabpy": [ "VERSION", - "tabpy_server/state.ini.template", - "tabpy_server/static/*", - "tabpy_server/common/default.conf", + "server/state.ini.template", + "server/static/*", + "server/common/default.conf", ] }, python_requires=">=3.6", diff --git a/tabpy/VERSION b/tabpy/VERSION index 9084fa2f..227cea21 100755 --- a/tabpy/VERSION +++ b/tabpy/VERSION @@ -1 +1 @@ -1.1.0 +2.0.0 diff --git a/tabpy/models/deploy_models.py b/tabpy/models/deploy_models.py index a41171a7..4aa54111 100644 --- a/tabpy/models/deploy_models.py +++ b/tabpy/models/deploy_models.py @@ -1,6 +1,5 @@ import os from pathlib import Path -import pip import platform import subprocess import sys @@ -9,26 +8,18 @@ def main(): # Determine if we run python or python3 - if platform.system() == "Windows": - py = "python" - else: - py = "python3" + py = "python" if platform.system() == "Windows" else "python3" - if len(sys.argv) > 1: - config_file_path = sys.argv[1] - else: - config_file_path = setup_utils.get_default_config_file_path() - print(f"Using config file at {config_file_path}") - port, auth_on, prefix = setup_utils.parse_config(config_file_path) - if auth_on: - auth_args = setup_utils.get_creds() - else: - auth_args = [] + file_path = sys.argv[1] if len(sys.argv) > 1 else setup_utils.get_default_config_file_path() + print(f"Using config file at {file_path}") + + port, auth_on, prefix = setup_utils.parse_config(file_path) + auth_args = setup_utils.get_creds() if auth_on else [] directory = str(Path(__file__).resolve().parent / "scripts") # Deploy each model in the scripts directory for filename in os.listdir(directory): - subprocess.run([py, f"{directory}/{filename}", config_file_path] + auth_args) + subprocess.run([py, f"{directory}/{filename}", file_path] + auth_args) if __name__ == "__main__": diff --git a/tabpy/models/utils/setup_utils.py b/tabpy/models/utils/setup_utils.py index cb69c8e5..5a3f8458 100644 --- a/tabpy/models/utils/setup_utils.py +++ b/tabpy/models/utils/setup_utils.py @@ -2,14 +2,14 @@ import getpass import os import sys -from tabpy.tabpy_tools.client import Client +from tabpy.tools.client import Client def get_default_config_file_path(): import tabpy pkg_path = os.path.dirname(tabpy.__file__) - config_file_path = os.path.join(pkg_path, "tabpy_server", "common", "default.conf") + config_file_path = os.path.join(pkg_path, "server", "common", "default.conf") return config_file_path @@ -44,21 +44,14 @@ def get_creds(): def deploy_model(funcName, func, funcDescription): # running from deploy_models.py - if len(sys.argv) > 1: - config_file_path = sys.argv[1] - else: - config_file_path = get_default_config_file_path() + config_file_path = sys.argv[1] if len(sys.argv) > 1 else get_default_config_file_path() port, auth_on, prefix = parse_config(config_file_path) connection = Client(f"{prefix}://localhost:{port}/") if auth_on: # credentials are passed in from setup.py - if len(sys.argv) == 4: - user, passwd = sys.argv[2], sys.argv[3] - # running Sentiment Analysis independently - else: - user, passwd = get_creds() + user, passwd = sys.argv[2], sys.argv[3] if len(sys.argv) == 4 else get_creds() connection.set_credentials(user, passwd) connection.deploy(funcName, func, funcDescription, override=True) diff --git a/tabpy/tabpy_server/__init__.py b/tabpy/server/__init__.py similarity index 100% rename from tabpy/tabpy_server/__init__.py rename to tabpy/server/__init__.py diff --git a/tabpy/tabpy_server/app/ConfigParameters.py b/tabpy/server/app/ConfigParameters.py similarity index 100% rename from tabpy/tabpy_server/app/ConfigParameters.py rename to tabpy/server/app/ConfigParameters.py diff --git a/tabpy/tabpy_server/app/SettingsParameters.py b/tabpy/server/app/SettingsParameters.py old mode 100755 new mode 100644 similarity index 100% rename from tabpy/tabpy_server/app/SettingsParameters.py rename to tabpy/server/app/SettingsParameters.py diff --git a/tabpy/tabpy_server/app/__init__.py b/tabpy/server/app/__init__.py similarity index 100% rename from tabpy/tabpy_server/app/__init__.py rename to tabpy/server/app/__init__.py diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/server/app/app.py similarity index 94% rename from tabpy/tabpy_server/app/app.py rename to tabpy/server/app/app.py index 563b1c65..4864b742 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/server/app/app.py @@ -1,22 +1,21 @@ import concurrent.futures import configparser import logging -from logging import config import multiprocessing import os import shutil import signal import sys -import tabpy.tabpy_server +import tabpy from tabpy.tabpy import __version__ -from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters -from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy.tabpy_server.app.util import parse_pwd_file -from tabpy.tabpy_server.management.state import TabPyState -from tabpy.tabpy_server.management.util import _get_state_from_file -from tabpy.tabpy_server.psws.callbacks import init_model_evaluator, init_ps_server -from tabpy.tabpy_server.psws.python_service import PythonService, PythonServiceHandler -from tabpy.tabpy_server.handlers import ( +from tabpy.server.app.ConfigParameters import ConfigParameters +from tabpy.server.app.SettingsParameters import SettingsParameters +from tabpy.server.app.util import parse_pwd_file +from tabpy.server.management.state import TabPyState +from tabpy.server.management.util import _get_state_from_file +from tabpy.server.psws.callbacks import init_model_evaluator, init_ps_server +from tabpy.server.psws.python_service import PythonService, PythonServiceHandler +from tabpy.server.handlers import ( EndpointHandler, EndpointsHandler, EvaluationPlaneHandler, @@ -69,7 +68,8 @@ def __init__(self, config_file=None): if os.path.isfile(config_file): try: - logging.config.fileConfig(config_file, disable_existing_loggers=False) + from logging import config + config.fileConfig(config_file, disable_existing_loggers=False) except KeyError: logging.basicConfig(level=logging.DEBUG) @@ -266,9 +266,9 @@ def _parse_config(self, config_file): None, None), (SettingsParameters.KeyFile, ConfigParameters.TABPY_KEY_FILE, None, None), (SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, - os.path.join(pkg_path, "tabpy_server"), None), + os.path.join(pkg_path, "server"), None), (SettingsParameters.StaticPath, ConfigParameters.TABPY_STATIC_PATH, - os.path.join(pkg_path, "tabpy_server", "static"), None), + os.path.join(pkg_path, "server", "static"), None), (ConfigParameters.TABPY_PWD_FILE, ConfigParameters.TABPY_PWD_FILE, None, None), (SettingsParameters.LogRequestContext, ConfigParameters.TABPY_LOG_DETAILS, "false", None), @@ -367,7 +367,7 @@ def _validate_transfer_protocol_settings(self): os.path.isfile(cert), os.path.isfile(self.settings[SettingsParameters.KeyFile]), ) - tabpy.tabpy_server.app.util.validate_cert(cert) + tabpy.server.app.util.validate_cert(cert) @staticmethod def _validate_cert_key_state(msg, cert_valid, key_valid): @@ -417,7 +417,7 @@ def _build_tabpy_state(self): state_file_path = os.path.join(state_file_dir, "state.ini") if not os.path.isfile(state_file_path): state_file_template_path = os.path.join( - pkg_path, "tabpy_server", "state.ini.template" + pkg_path, "server", "state.ini.template" ) logger.debug( f"File {state_file_path} not found, creating from " diff --git a/tabpy/tabpy_server/app/util.py b/tabpy/server/app/util.py similarity index 100% rename from tabpy/tabpy_server/app/util.py rename to tabpy/server/app/util.py diff --git a/tabpy/tabpy_server/common/__init__.py b/tabpy/server/common/__init__.py similarity index 100% rename from tabpy/tabpy_server/common/__init__.py rename to tabpy/server/common/__init__.py diff --git a/tabpy/tabpy_server/common/default.conf b/tabpy/server/common/default.conf old mode 100755 new mode 100644 similarity index 94% rename from tabpy/tabpy_server/common/default.conf rename to tabpy/server/common/default.conf index ee02453d..80e9dd49 --- a/tabpy/tabpy_server/common/default.conf +++ b/tabpy/server/common/default.conf @@ -1,10 +1,10 @@ [TabPy] # TABPY_QUERY_OBJECT_PATH = /tmp/query_objects # TABPY_PORT = 9004 -# TABPY_STATE_PATH = ./tabpy/tabpy_server +# TABPY_STATE_PATH = ./tabpy/server # Where static pages live -# TABPY_STATIC_PATH = ./tabpy/tabpy_server/static +# TABPY_STATIC_PATH = ./tabpy/server/static # For how to configure TabPy authentication read # Authentication section in docs/server-config.md. diff --git a/tabpy/tabpy_server/common/endpoint_file_mgr.py b/tabpy/server/common/endpoint_file_mgr.py similarity index 100% rename from tabpy/tabpy_server/common/endpoint_file_mgr.py rename to tabpy/server/common/endpoint_file_mgr.py diff --git a/tabpy/tabpy_server/common/messages.py b/tabpy/server/common/messages.py similarity index 100% rename from tabpy/tabpy_server/common/messages.py rename to tabpy/server/common/messages.py diff --git a/tabpy/tabpy_server/common/util.py b/tabpy/server/common/util.py similarity index 100% rename from tabpy/tabpy_server/common/util.py rename to tabpy/server/common/util.py diff --git a/tabpy/server/handlers/__init__.py b/tabpy/server/handlers/__init__.py new file mode 100644 index 00000000..fbf68bd7 --- /dev/null +++ b/tabpy/server/handlers/__init__.py @@ -0,0 +1,13 @@ +from tabpy.server.handlers.base_handler import BaseHandler +from tabpy.server.handlers.main_handler import MainHandler +from tabpy.server.handlers.management_handler import ManagementHandler + +from tabpy.server.handlers.endpoint_handler import EndpointHandler +from tabpy.server.handlers.endpoints_handler import EndpointsHandler +from tabpy.server.handlers.evaluation_plane_handler import EvaluationPlaneHandler +from tabpy.server.handlers.query_plane_handler import QueryPlaneHandler +from tabpy.server.handlers.service_info_handler import ServiceInfoHandler +from tabpy.server.handlers.status_handler import StatusHandler +from tabpy.server.handlers.upload_destination_handler import ( + UploadDestinationHandler, +) diff --git a/tabpy/tabpy_server/handlers/base_handler.py b/tabpy/server/handlers/base_handler.py similarity index 96% rename from tabpy/tabpy_server/handlers/base_handler.py rename to tabpy/server/handlers/base_handler.py index dbbb6371..853a00a5 100644 --- a/tabpy/tabpy_server/handlers/base_handler.py +++ b/tabpy/server/handlers/base_handler.py @@ -4,8 +4,8 @@ import json import logging import tornado.web -from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy.tabpy_server.handlers.util import hash_password +from tabpy.server.app.SettingsParameters import SettingsParameters +from tabpy.server.handlers.util import hash_password import uuid @@ -38,15 +38,8 @@ def set_request(self, request: tornado.httputil.HTTPServerRequest): self.method = request.method self.url = request.full_url() - if "TabPy-Client" in request.headers: - self.client = request.headers["TabPy-Client"] - else: - self.client = None - - if "TabPy-User" in request.headers: - self.tableau_username = request.headers["TabPy-User"] - else: - self.tableau_username = None + self.client = request.headers.get("TabPy-Client", None) + self.tableau_username = request.headers.get("TabPy-User", None) def set_tabpy_username(self, tabpy_username: str): self.tabpy_username = tabpy_username diff --git a/tabpy/tabpy_server/handlers/endpoint_handler.py b/tabpy/server/handlers/endpoint_handler.py similarity index 93% rename from tabpy/tabpy_server/handlers/endpoint_handler.py rename to tabpy/server/handlers/endpoint_handler.py index 022d8e0b..2f882ef8 100644 --- a/tabpy/tabpy_server/handlers/endpoint_handler.py +++ b/tabpy/server/handlers/endpoint_handler.py @@ -9,11 +9,11 @@ import json import logging import shutil -from tabpy.tabpy_server.common.util import format_exception -from tabpy.tabpy_server.handlers import ManagementHandler -from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD -from tabpy.tabpy_server.management.state import get_query_object_path -from tabpy.tabpy_server.psws.callbacks import on_state_change +from tabpy.server.common.util import format_exception +from tabpy.server.handlers import ManagementHandler +from tabpy.server.handlers.base_handler import STAGING_THREAD +from tabpy.server.management.state import get_query_object_path +from tabpy.server.psws.callbacks import on_state_change from tornado import gen diff --git a/tabpy/tabpy_server/handlers/endpoints_handler.py b/tabpy/server/handlers/endpoints_handler.py similarity index 95% rename from tabpy/tabpy_server/handlers/endpoints_handler.py rename to tabpy/server/handlers/endpoints_handler.py index 66132dd2..0bd17c16 100644 --- a/tabpy/tabpy_server/handlers/endpoints_handler.py +++ b/tabpy/server/handlers/endpoints_handler.py @@ -8,8 +8,8 @@ import json import logging -from tabpy.tabpy_server.common.util import format_exception -from tabpy.tabpy_server.handlers import ManagementHandler +from tabpy.server.common.util import format_exception +from tabpy.server.handlers import ManagementHandler from tornado import gen diff --git a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy/server/handlers/evaluation_plane_handler.py similarity index 91% rename from tabpy/tabpy_server/handlers/evaluation_plane_handler.py rename to tabpy/server/handlers/evaluation_plane_handler.py index 390aff04..3e73e359 100644 --- a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy/server/handlers/evaluation_plane_handler.py @@ -1,8 +1,8 @@ -from tabpy.tabpy_server.handlers import BaseHandler +from tabpy.server.handlers import BaseHandler import json import simplejson import logging -from tabpy.tabpy_server.common.util import format_exception +from tabpy.server.common.util import format_exception import requests from tornado import gen from datetime import timedelta @@ -134,12 +134,9 @@ def _call_subprocess(self, function_to_evaluate, arguments): # name - the name is actually defined with user script being wrapped # in _user_script function (constructed as a striong) and then executed # with exec() call above. - if arguments is None: - future = self.executor.submit(_user_script, # noqa: F821 - restricted_tabpy) - else: - future = self.executor.submit(_user_script, # noqa: F821 - restricted_tabpy, **arguments) + future = self.executor.submit(_user_script, # noqa: F821 + restricted_tabpy, + **arguments if arguments is not None else None) ret = yield gen.with_timeout(timedelta(seconds=self.eval_timeout), future) raise gen.Return(ret) diff --git a/tabpy/tabpy_server/handlers/main_handler.py b/tabpy/server/handlers/main_handler.py similarity index 70% rename from tabpy/tabpy_server/handlers/main_handler.py rename to tabpy/server/handlers/main_handler.py index dbf2680b..d6a43025 100644 --- a/tabpy/tabpy_server/handlers/main_handler.py +++ b/tabpy/server/handlers/main_handler.py @@ -1,4 +1,4 @@ -from tabpy.tabpy_server.handlers import BaseHandler +from tabpy.server.handlers import BaseHandler class MainHandler(BaseHandler): diff --git a/tabpy/tabpy_server/handlers/management_handler.py b/tabpy/server/handlers/management_handler.py similarity index 71% rename from tabpy/tabpy_server/handlers/management_handler.py rename to tabpy/server/handlers/management_handler.py index 90e8a541..b24812dd 100644 --- a/tabpy/tabpy_server/handlers/management_handler.py +++ b/tabpy/server/handlers/management_handler.py @@ -6,11 +6,11 @@ from tornado import gen -from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy.tabpy_server.handlers import MainHandler -from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD -from tabpy.tabpy_server.management.state import get_query_object_path -from tabpy.tabpy_server.psws.callbacks import on_state_change +from tabpy.server.app.SettingsParameters import SettingsParameters +from tabpy.server.handlers import MainHandler +from tabpy.server.handlers.base_handler import STAGING_THREAD +from tabpy.server.management.state import get_query_object_path +from tabpy.server.psws.callbacks import on_state_change def copy_from_local(localpath, remotepath, is_dir=False): @@ -49,13 +49,13 @@ def _add_or_update_endpoint(self, action, name, version, request_data): """ self.logger.log(logging.DEBUG, f"Adding/updating model {name}...") - _name_checker = _compile(r"^[a-zA-Z0-9-_\s]+$") if not isinstance(name, str): msg = "Endpoint name must be a string" self.logger.log(logging.CRITICAL, msg) raise TypeError(msg) - if not _name_checker.match(name): + name_checker = _compile(r"^[a-zA-Z0-9-_\s]+$") + if not name_checker.match(name): raise gen.Return( "endpoint name can only contain: a-z, A-Z, 0-9," " underscore, hyphens and spaces." @@ -69,48 +69,38 @@ def _add_or_update_endpoint(self, action, name, version, request_data): self.logger.log(logging.CRITICAL, msg) raise RuntimeError(msg) - request_uuid = random_uuid() - self.settings["add_or_updating_endpoint"] = request_uuid + self.settings["add_or_updating_endpoint"] = random_uuid() try: - description = ( - request_data["description"] if "description" in request_data else None - ) + docstring = None if "docstring" in request_data: docstring = str( bytes(request_data["docstring"], "utf-8").decode("unicode_escape") ) - else: - docstring = None - endpoint_type = request_data["type"] if "type" in request_data else None - methods = request_data["methods"] if "methods" in request_data else [] - dependencies = ( - request_data["dependencies"] if "dependencies" in request_data else None - ) - target = request_data["target"] if "target" in request_data else None - schema = request_data["schema"] if "schema" in request_data else None - src_path = request_data["src_path"] if "src_path" in request_data else None + description = request_data.get("description", None) + endpoint_type = request_data.get("type", None) + methods = request_data.get("methods", []) + dependencies = request_data.get("dependencies", None) + target = request_data.get("target", None) + schema = request_data.get("schema", None) + src_path = request_data.get("src_path", None) target_path = get_query_object_path( self.settings[SettingsParameters.StateFilePath], name, version ) - self.logger.log(logging.DEBUG, f"Checking source path {src_path}...") - _path_checker = _compile(r"^[\\\:a-zA-Z0-9-_~\s/\.\(\)]+$") + + path_checker = _compile(r"^[\\\:a-zA-Z0-9-_~\s/\.\(\)]+$") # copy from staging if src_path: - if not isinstance(request_data["src_path"], str): + if not isinstance(src_path, str): raise gen.Return("src_path must be a string.") - if not _path_checker.match(src_path): - raise gen.Return( - "Endpoint source path name can only contain: " - "a-z, A-Z, 0-9, underscore, hyphens and spaces." - ) + if not path_checker.match(src_path): + raise gen.Return(f"Invalid source path for endpoint {name}") yield self._copy_po_future(src_path, target_path) elif endpoint_type != "alias": - raise gen.Return("src_path is required to add/update an " "endpoint.") - - # alias special logic: - if endpoint_type == "alias": + raise gen.Return("src_path is required to add/update an endpoint.") + else: + # alias special logic: if not target: raise gen.Return("Target is required for alias endpoint.") dependencies = [target] diff --git a/tabpy/tabpy_server/handlers/query_plane_handler.py b/tabpy/server/handlers/query_plane_handler.py similarity index 97% rename from tabpy/tabpy_server/handlers/query_plane_handler.py rename to tabpy/server/handlers/query_plane_handler.py index aab42593..16928b4f 100644 --- a/tabpy/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy/server/handlers/query_plane_handler.py @@ -1,7 +1,7 @@ -from tabpy.tabpy_server.handlers import BaseHandler +from tabpy.server.handlers import BaseHandler import logging import time -from tabpy.tabpy_server.common.messages import ( +from tabpy.server.common.messages import ( Query, QuerySuccessful, QueryError, @@ -10,7 +10,7 @@ from hashlib import md5 import uuid import json -from tabpy.tabpy_server.common.util import format_exception +from tabpy.server.common.util import format_exception import urllib from tornado import gen diff --git a/tabpy/tabpy_server/handlers/service_info_handler.py b/tabpy/server/handlers/service_info_handler.py similarity index 85% rename from tabpy/tabpy_server/handlers/service_info_handler.py rename to tabpy/server/handlers/service_info_handler.py index 51152a98..2451d199 100644 --- a/tabpy/tabpy_server/handlers/service_info_handler.py +++ b/tabpy/server/handlers/service_info_handler.py @@ -1,6 +1,6 @@ import json -from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy.tabpy_server.handlers import ManagementHandler +from tabpy.server.app.SettingsParameters import SettingsParameters +from tabpy.server.handlers import ManagementHandler class ServiceInfoHandler(ManagementHandler): diff --git a/tabpy/tabpy_server/handlers/status_handler.py b/tabpy/server/handlers/status_handler.py similarity index 93% rename from tabpy/tabpy_server/handlers/status_handler.py rename to tabpy/server/handlers/status_handler.py index 2f743b3f..622e0199 100644 --- a/tabpy/tabpy_server/handlers/status_handler.py +++ b/tabpy/server/handlers/status_handler.py @@ -1,6 +1,6 @@ import json import logging -from tabpy.tabpy_server.handlers import BaseHandler +from tabpy.server.handlers import BaseHandler class StatusHandler(BaseHandler): diff --git a/tabpy/tabpy_server/handlers/upload_destination_handler.py b/tabpy/server/handlers/upload_destination_handler.py similarity index 79% rename from tabpy/tabpy_server/handlers/upload_destination_handler.py rename to tabpy/server/handlers/upload_destination_handler.py index 729aff3e..a04dad8e 100644 --- a/tabpy/tabpy_server/handlers/upload_destination_handler.py +++ b/tabpy/server/handlers/upload_destination_handler.py @@ -1,5 +1,5 @@ -from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy.tabpy_server.handlers import ManagementHandler +from tabpy.server.app.SettingsParameters import SettingsParameters +from tabpy.server.handlers import ManagementHandler import os diff --git a/tabpy/tabpy_server/handlers/util.py b/tabpy/server/handlers/util.py old mode 100755 new mode 100644 similarity index 96% rename from tabpy/tabpy_server/handlers/util.py rename to tabpy/server/handlers/util.py index 3d1cff1b..14e029c6 --- a/tabpy/tabpy_server/handlers/util.py +++ b/tabpy/server/handlers/util.py @@ -1,32 +1,32 @@ -import binascii -from hashlib import pbkdf2_hmac - - -def hash_password(username, pwd): - """ - Hashes password using PKDBF2 method: - hash = PKDBF2('sha512', pwd, salt=username, 10000) - - Parameters - ---------- - username : str - User name (login). Used as salt for hashing. - User name is lowercased befor being used in hashing. - Salt is formatted as '_$salt@tabpy:$_' to - guarantee there's at least 16 characters. - - pwd : str - Password to hash. - - Returns - ------- - str - Sting representation (hexidecimal) for PBKDF2 hash - for the password. - """ - salt = f"_$salt@tabpy:{username.lower()}$_" - - hash = pbkdf2_hmac( - hash_name="sha512", password=pwd.encode(), salt=salt.encode(), iterations=10000 - ) - return binascii.hexlify(hash).decode() +import binascii +from hashlib import pbkdf2_hmac + + +def hash_password(username, pwd): + """ + Hashes password using PKDBF2 method: + hash = PKDBF2('sha512', pwd, salt=username, 10000) + + Parameters + ---------- + username : str + User name (login). Used as salt for hashing. + User name is lowercased befor being used in hashing. + Salt is formatted as '_$salt@tabpy:$_' to + guarantee there's at least 16 characters. + + pwd : str + Password to hash. + + Returns + ------- + str + Sting representation (hexidecimal) for PBKDF2 hash + for the password. + """ + salt = f"_$salt@tabpy:{username.lower()}$_" + + hash = pbkdf2_hmac( + hash_name="sha512", password=pwd.encode(), salt=salt.encode(), iterations=10000 + ) + return binascii.hexlify(hash).decode() diff --git a/tabpy/tabpy_server/management/__init__.py b/tabpy/server/management/__init__.py similarity index 100% rename from tabpy/tabpy_server/management/__init__.py rename to tabpy/server/management/__init__.py diff --git a/tabpy/tabpy_server/management/state.py b/tabpy/server/management/state.py similarity index 85% rename from tabpy/tabpy_server/management/state.py rename to tabpy/server/management/state.py index c36f7710..697d24fd 100644 --- a/tabpy/tabpy_server/management/state.py +++ b/tabpy/server/management/state.py @@ -4,7 +4,7 @@ from configparser import ConfigParser import json import logging -from tabpy.tabpy_server.management.util import write_state_config +from tabpy.server.management.util import write_state_config from threading import Lock from time import time @@ -44,9 +44,9 @@ def wrapper(self, *args, **kwargs): def _get_root_path(state_path): if state_path[-1] != "/": - return state_path + "/" - else: - return state_path + state_path += "/" + + return state_path def get_query_object_path(state_file_path, name, version): @@ -56,10 +56,10 @@ def get_query_object_path(state_file_path, name, version): If the version is None, a path without the version will be returned. """ root_path = _get_root_path(state_file_path) + sub_path = [_QUERY_OBJECT_DIR, name] if version is not None: - full_path = root_path + "/".join([_QUERY_OBJECT_DIR, name, str(version)]) - else: - full_path = root_path + "/".join([_QUERY_OBJECT_DIR, name]) + sub_path.append(str(version)) + full_path = root_path + "/".join(sub_path) return full_path @@ -148,6 +148,45 @@ def get_endpoints(self, name=None): logger.debug(f"Collected endpoints: {endpoints}") return endpoints + def _check_endpoint_exists(self, name): + endpoints = self.get_endpoints() + if not name or not isinstance(name, str) or len(name) == 0: + raise ValueError("name of the endpoint must be a valid string.") + + return name in endpoints + + def _check_and_set_endpoint_str_value(self, param, paramName, defaultValue): + if not param and defaultValue is not None: + return defaultValue + + if not param or not isinstance(param, str): + raise ValueError(f"{paramName} must be a string.") + + return param + + def _check_and_set_endpoint_description(self, description, defaultValue): + return self._check_and_set_endpoint_str_value(description, "description", defaultValue) + + def _check_and_set_endpoint_docstring(self, docstring, defaultValue): + return self._check_and_set_endpoint_str_value(docstring, "docstring", defaultValue) + + def _check_and_set_endpoint_type(self, endpoint_type, defaultValue): + return self._check_and_set_endpoint_str_value( + endpoint_type, "endpoint type", defaultValue) + + def _check_and_set_target(self, target, defaultValue): + return self._check_and_set_endpoint_str_value( + target, "target", defaultValue) + + def _check_and_set_dependencies(self, dependencies, defaultValue): + if not dependencies: + return defaultValue + + if dependencies or not isinstance(dependencies, list): + raise ValueError("dependencies must be a list.") + + return dependencies + @state_lock def add_endpoint( self, @@ -182,28 +221,19 @@ def add_endpoint( """ try: - endpoints = self.get_endpoints() - if name is None or not isinstance(name, str) or len(name) == 0: - raise ValueError("name of the endpoint must be a valid string.") - elif name in endpoints: + if (self._check_endpoint_exists(name)): raise ValueError(f"endpoint {name} already exists.") - if description and not isinstance(description, str): - raise ValueError("description must be a string.") - elif not description: - description = "" - if docstring and not isinstance(docstring, str): - raise ValueError("docstring must be a string.") - elif not docstring: - docstring = "-- no docstring found in query function --" - if not endpoint_type or not isinstance(endpoint_type, str): - raise ValueError("endpoint type must be a string.") - if dependencies and not isinstance(dependencies, list): - raise ValueError("dependencies must be a list.") - elif not dependencies: - dependencies = [] - if target and not isinstance(target, str): - raise ValueError("target must be a string.") - elif target and target not in endpoints: + + endpoints = self.get_endpoints() + + description = self._check_and_set_endpoint_description(description, "") + docstring = self._check_and_set_endpoint_docstring( + docstring, "-- no docstring found in query function --") + endpoint_type = self._check_and_set_endpoint_type(endpoint_type, None) + dependencies = self._check_and_set_dependencies(dependencies, []) + + target = self._check_and_set_target(target, "") + if target and target not in endpoints: raise ValueError("target endpoint is not valid.") endpoint_info = { @@ -286,43 +316,32 @@ def update_endpoint( """ try: - endpoints = self.get_endpoints() - if not name or not isinstance(name, str): - raise ValueError("name of the endpoint must be string.") - elif name not in endpoints: + if (not self._check_endpoint_exists(name)): raise ValueError(f"endpoint {name} does not exist.") + endpoints = self.get_endpoints() endpoint_info = endpoints[name] - if description and not isinstance(description, str): - raise ValueError("description must be a string.") - elif not description: - description = endpoint_info["description"] - if docstring and not isinstance(docstring, str): - raise ValueError("docstring must be a string.") - elif not docstring: - docstring = endpoint_info["docstring"] - if endpoint_type and not isinstance(endpoint_type, str): - raise ValueError("endpoint type must be a string.") - elif not endpoint_type: - endpoint_type = endpoint_info["type"] + description = self._check_and_set_endpoint_description( + description, endpoint_info["description"]) + docstring = self._check_and_set_endpoint_docstring( + docstring, endpoint_info["docstring"]) + endpoint_type = self._check_and_set_endpoint_type( + endpoint_type, endpoint_info["type"]) + dependencies = self._check_and_set_dependencies( + dependencies, endpoint_info.get("dependencies", [])) + + target = self._check_and_set_target(target, None) + if target and target not in endpoints: + raise ValueError("target endpoint is not valid.") + elif not target: + target = endpoint_info["target"] + if version and not isinstance(version, int): raise ValueError("version must be an int.") elif not version: version = endpoint_info["version"] - if dependencies and not isinstance(dependencies, list): - raise ValueError("dependencies must be a list.") - elif not dependencies: - if "dependencies" in endpoint_info: - dependencies = endpoint_info["dependencies"] - else: - dependencies = [] - if target and not isinstance(target, str): - raise ValueError("target must be a string.") - elif target and target not in endpoints: - raise ValueError("target endpoint is not valid.") - elif not target: - target = endpoint_info["target"] + endpoint_info = { "description": description, "docstring": docstring, diff --git a/tabpy/tabpy_server/management/util.py b/tabpy/server/management/util.py similarity index 90% rename from tabpy/tabpy_server/management/util.py rename to tabpy/server/management/util.py index cb9b7709..8631530f 100644 --- a/tabpy/tabpy_server/management/util.py +++ b/tabpy/server/management/util.py @@ -5,8 +5,8 @@ from ConfigParser import ConfigParser as _ConfigParser except ImportError: from configparser import ConfigParser as _ConfigParser -from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters -from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.server.app.ConfigParameters import ConfigParameters +from tabpy.server.app.SettingsParameters import SettingsParameters def write_state_config(state, settings, logger=logging.getLogger(__name__)): diff --git a/tabpy/tabpy_server/psws/__init__.py b/tabpy/server/psws/__init__.py similarity index 100% rename from tabpy/tabpy_server/psws/__init__.py rename to tabpy/server/psws/__init__.py diff --git a/tabpy/tabpy_server/psws/callbacks.py b/tabpy/server/psws/callbacks.py similarity index 94% rename from tabpy/tabpy_server/psws/callbacks.py rename to tabpy/server/psws/callbacks.py index 4b1fe14e..22290b03 100644 --- a/tabpy/tabpy_server/psws/callbacks.py +++ b/tabpy/server/psws/callbacks.py @@ -1,15 +1,15 @@ import logging -from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters -from tabpy.tabpy_server.common.messages import ( +from tabpy.server.app.SettingsParameters import SettingsParameters +from tabpy.server.common.messages import ( LoadObject, DeleteObjects, ListObjects, ObjectList, ) -from tabpy.tabpy_server.common.endpoint_file_mgr import cleanup_endpoint_files -from tabpy.tabpy_server.common.util import format_exception -from tabpy.tabpy_server.management.state import TabPyState, get_query_object_path -from tabpy.tabpy_server.management import util +from tabpy.server.common.endpoint_file_mgr import cleanup_endpoint_files +from tabpy.server.common.util import format_exception +from tabpy.server.management.state import TabPyState, get_query_object_path +from tabpy.server.management import util from time import sleep from tornado import gen diff --git a/tabpy/tabpy_server/psws/python_service.py b/tabpy/server/psws/python_service.py similarity index 98% rename from tabpy/tabpy_server/psws/python_service.py rename to tabpy/server/psws/python_service.py index 5352b40b..877dc051 100644 --- a/tabpy/tabpy_server/psws/python_service.py +++ b/tabpy/server/psws/python_service.py @@ -1,8 +1,7 @@ import concurrent.futures import logging -from tabpy.tabpy_tools.query_object import QueryObject -from tabpy.tabpy_server.common.util import format_exception -from tabpy.tabpy_server.common.messages import ( +from tabpy.server.common.util import format_exception +from tabpy.server.common.messages import ( LoadObject, DeleteObjects, FlushObjects, @@ -20,6 +19,7 @@ ObjectCount, ObjectList, ) +from tabpy.tools.query_object import QueryObject logger = logging.getLogger(__name__) diff --git a/tabpy/tabpy_server/state.ini.template b/tabpy/server/state.ini.template old mode 100755 new mode 100644 similarity index 94% rename from tabpy/tabpy_server/state.ini.template rename to tabpy/server/state.ini.template index b3828973..8999537f --- a/tabpy/tabpy_server/state.ini.template +++ b/tabpy/server/state.ini.template @@ -1,15 +1,15 @@ -[Service Info] -Name = TabPy Server -Description = -Creation Time = 0 -Access-Control-Allow-Origin = -Access-Control-Allow-Headers = -Access-Control-Allow-Methods = - -[Query Objects Service Versions] - -[Query Objects Docstrings] - -[Meta] -Revision Number = 1 - +[Service Info] +Name = TabPy Server +Description = +Creation Time = 0 +Access-Control-Allow-Origin = +Access-Control-Allow-Headers = +Access-Control-Allow-Methods = + +[Query Objects Service Versions] + +[Query Objects Docstrings] + +[Meta] +Revision Number = 1 + diff --git a/tabpy/tabpy_server/static/index.html b/tabpy/server/static/index.html similarity index 100% rename from tabpy/tabpy_server/static/index.html rename to tabpy/server/static/index.html diff --git a/tabpy/tabpy_server/static/tableau.png b/tabpy/server/static/tableau.png similarity index 100% rename from tabpy/tabpy_server/static/tableau.png rename to tabpy/server/static/tableau.png diff --git a/tabpy/tabpy.py b/tabpy/tabpy.py index 50acf799..bf56c0e1 100755 --- a/tabpy/tabpy.py +++ b/tabpy/tabpy.py @@ -38,7 +38,7 @@ def main(): args = docopt.docopt(__doc__) config = args["--config"] or None - from tabpy.tabpy_server.app.app import TabPyApp + from tabpy.server.app.app import TabPyApp app = TabPyApp(config) app.run() diff --git a/tabpy/tabpy_server/handlers/__init__.py b/tabpy/tabpy_server/handlers/__init__.py deleted file mode 100644 index 0c00cde6..00000000 --- a/tabpy/tabpy_server/handlers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from tabpy.tabpy_server.handlers.base_handler import BaseHandler -from tabpy.tabpy_server.handlers.main_handler import MainHandler -from tabpy.tabpy_server.handlers.management_handler import ManagementHandler - -from tabpy.tabpy_server.handlers.endpoint_handler import EndpointHandler -from tabpy.tabpy_server.handlers.endpoints_handler import EndpointsHandler -from tabpy.tabpy_server.handlers.evaluation_plane_handler import EvaluationPlaneHandler -from tabpy.tabpy_server.handlers.query_plane_handler import QueryPlaneHandler -from tabpy.tabpy_server.handlers.service_info_handler import ServiceInfoHandler -from tabpy.tabpy_server.handlers.status_handler import StatusHandler -from tabpy.tabpy_server.handlers.upload_destination_handler import ( - UploadDestinationHandler, -) diff --git a/tabpy/tabpy_tools/__init__.py b/tabpy/tools/__init__.py old mode 100755 new mode 100644 similarity index 100% rename from tabpy/tabpy_tools/__init__.py rename to tabpy/tools/__init__.py diff --git a/tabpy/tabpy_tools/client.py b/tabpy/tools/client.py similarity index 96% rename from tabpy/tabpy_tools/client.py rename to tabpy/tools/client.py index 96c2c267..f9e90023 100644 --- a/tabpy/tabpy_tools/client.py +++ b/tabpy/tools/client.py @@ -77,10 +77,9 @@ def __init__(self, endpoint, query_timeout=1000): service_client = ServiceClient(self._endpoint, network_wrapper) self._service = RESTServiceClient(service_client) - if type(query_timeout) in (int, float) and query_timeout > 0: - self._service.query_timeout = query_timeout - else: - self._service.query_timeout = 0.0 + if not type(query_timeout) in (int, float) or query_timeout <= 0: + query_timeout = 0.0 + self._service.query_timeout = query_timeout def __repr__(self): return ( @@ -226,6 +225,7 @@ def deploy(self, name, obj, description="", schema=None, override=False): remove, get_endpoints """ endpoint = self.get_endpoints().get(name) + version = 1 if endpoint: if not override: raise RuntimeError( @@ -235,8 +235,6 @@ def deploy(self, name, obj, description="", schema=None, override=False): ) version = endpoint.version + 1 - else: - version = 1 obj = self._gen_endpoint(name, obj, description, version, schema) @@ -305,11 +303,7 @@ def _gen_endpoint(self, name, obj, description, version=1, schema=None): _check_endpoint_name(name) if description is None: - if isinstance(obj.__doc__, str): - # extract doc string - description = obj.__doc__.strip() or "" - else: - description = "" + description = obj.__doc__.strip() or "" if isinstance(obj.__doc__, str) else "" endpoint_object = CustomQueryObject(query=obj, description=description,) diff --git a/tabpy/tabpy_tools/custom_query_object.py b/tabpy/tools/custom_query_object.py old mode 100755 new mode 100644 similarity index 100% rename from tabpy/tabpy_tools/custom_query_object.py rename to tabpy/tools/custom_query_object.py diff --git a/tabpy/tabpy_tools/query_object.py b/tabpy/tools/query_object.py old mode 100755 new mode 100644 similarity index 100% rename from tabpy/tabpy_tools/query_object.py rename to tabpy/tools/query_object.py diff --git a/tabpy/tabpy_tools/rest.py b/tabpy/tools/rest.py old mode 100755 new mode 100644 similarity index 100% rename from tabpy/tabpy_tools/rest.py rename to tabpy/tools/rest.py diff --git a/tabpy/tabpy_tools/rest_client.py b/tabpy/tools/rest_client.py old mode 100755 new mode 100644 similarity index 100% rename from tabpy/tabpy_tools/rest_client.py rename to tabpy/tools/rest_client.py diff --git a/tabpy/tabpy_tools/schema.py b/tabpy/tools/schema.py old mode 100755 new mode 100644 similarity index 98% rename from tabpy/tabpy_tools/schema.py rename to tabpy/tools/schema.py index ba36bae2..27b2e425 --- a/tabpy/tabpy_tools/schema.py +++ b/tabpy/tools/schema.py @@ -73,7 +73,7 @@ def generate_schema(input, output, input_description=None, output_description=No ---------- .. sourcecode:: python For just one input parameter, state the example directly. - >>> from tabpy_tools.schema import generate_schema + >>> from tabpy.tools.schema import generate_schema >>> schema = generate_schema( input=5, output=25, diff --git a/tabpy/utils/tabpy_user.py b/tabpy/utils/tabpy_user.py index 065d0298..0039b84d 100755 --- a/tabpy/utils/tabpy_user.py +++ b/tabpy/utils/tabpy_user.py @@ -19,8 +19,8 @@ import docopt import logging import secrets -from tabpy.tabpy_server.app.util import parse_pwd_file -from tabpy.tabpy_server.handlers.util import hash_password +from tabpy.server.app.util import parse_pwd_file +from tabpy.server.handlers.util import hash_password logger = logging.getLogger(__name__) diff --git a/tests/integration/resources/deploy_and_evaluate_model.conf b/tests/integration/resources/deploy_and_evaluate_model.conf index 375ea47a..4b1ab753 100755 --- a/tests/integration/resources/deploy_and_evaluate_model.conf +++ b/tests/integration/resources/deploy_and_evaluate_model.conf @@ -1,10 +1,10 @@ [TabPy] # TABPY_QUERY_OBJECT_PATH = /tmp/query_objects TABPY_PORT = 9008 -# TABPY_STATE_PATH = ./tabpy/tabpy_server +# TABPY_STATE_PATH = ./tabpy/server # Where static pages live -# TABPY_STATIC_PATH = ./tabpy/tabpy_server/static +# TABPY_STATIC_PATH = ./tabpy/server/static # For how to configure TabPy authentication read # Authentication section in docs/server-config.md. diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 41e5d814..13448334 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -2,8 +2,8 @@ import unittest from tempfile import NamedTemporaryFile import tabpy -from tabpy.tabpy_server.app.util import validate_cert -from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.server.app.util import validate_cert +from tabpy.server.app.app import TabPyApp from unittest.mock import patch @@ -22,11 +22,11 @@ def test_config_file_does_not_exist(self): self.assertEqual(app.settings["log_request_context"], False) self.assertEqual(app.settings["evaluate_timeout"], 30) - @patch("tabpy.tabpy_server.app.app.TabPyState") - @patch("tabpy.tabpy_server.app.app._get_state_from_file") - @patch("tabpy.tabpy_server.app.app.PythonServiceHandler") - @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) - @patch("tabpy.tabpy_server.app.app.os") + @patch("tabpy.server.app.app.TabPyState") + @patch("tabpy.server.app.app._get_state_from_file") + @patch("tabpy.server.app.app.PythonServiceHandler") + @patch("tabpy.server.app.app.os.path.exists", return_value=True) + @patch("tabpy.server.app.app.os") def test_no_config_file( self, mock_os, @@ -37,7 +37,7 @@ def test_no_config_file( ): pkg_path = os.path.dirname(tabpy.__file__) obj_path = os.path.join(pkg_path, "tmp", "query_objects") - state_path = os.path.join(pkg_path, "tabpy_server") + state_path = os.path.join(pkg_path, "server") mock_os.environ = { "TABPY_PORT": "9004", "TABPY_QUERY_OBJECT_PATH": obj_path, @@ -52,11 +52,11 @@ def test_no_config_file( self.assertTrue(len(mock_management_util.mock_calls) > 0) mock_os.makedirs.assert_not_called() - @patch("tabpy.tabpy_server.app.app.TabPyState") - @patch("tabpy.tabpy_server.app.app._get_state_from_file") - @patch("tabpy.tabpy_server.app.app.PythonServiceHandler") - @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=False) - @patch("tabpy.tabpy_server.app.app.os") + @patch("tabpy.server.app.app.TabPyState") + @patch("tabpy.server.app.app._get_state_from_file") + @patch("tabpy.server.app.app.PythonServiceHandler") + @patch("tabpy.server.app.app.os.path.exists", return_value=False) + @patch("tabpy.server.app.app.os") def test_no_state_ini_file_or_state_dir( self, mock_os, @@ -77,11 +77,11 @@ def tearDown(self): os.remove(self.config_file.name) self.config_file = None - @patch("tabpy.tabpy_server.app.app.TabPyState") - @patch("tabpy.tabpy_server.app.app._get_state_from_file") - @patch("tabpy.tabpy_server.app.app.PythonServiceHandler") - @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) - @patch("tabpy.tabpy_server.app.app.os") + @patch("tabpy.server.app.app.TabPyState") + @patch("tabpy.server.app.app._get_state_from_file") + @patch("tabpy.server.app.app.PythonServiceHandler") + @patch("tabpy.server.app.app.os.path.exists", return_value=True) + @patch("tabpy.server.app.app.os") def test_config_file_present( self, mock_os, @@ -116,9 +116,9 @@ def test_config_file_present( self.assertEqual(app.settings["log_request_context"], False) self.assertEqual(app.settings["evaluate_timeout"], 30) - @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) - @patch("tabpy.tabpy_server.app.app._get_state_from_file") - @patch("tabpy.tabpy_server.app.app.TabPyState") + @patch("tabpy.server.app.app.os.path.exists", return_value=True) + @patch("tabpy.server.app.app._get_state_from_file") + @patch("tabpy.server.app.app.TabPyState") def test_custom_evaluate_timeout_valid( self, mock_state, mock_get_state_from_file, mock_path_exists ): @@ -130,9 +130,9 @@ def test_custom_evaluate_timeout_valid( app = TabPyApp(self.config_file.name) self.assertEqual(app.settings["evaluate_timeout"], 1996.0) - @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) - @patch("tabpy.tabpy_server.app.app._get_state_from_file") - @patch("tabpy.tabpy_server.app.app.TabPyState") + @patch("tabpy.server.app.app.os.path.exists", return_value=True) + @patch("tabpy.server.app.app._get_state_from_file") + @patch("tabpy.server.app.app.TabPyState") def test_custom_evaluate_timeout_invalid( self, mock_state, mock_get_state_from_file, mock_path_exists ): @@ -143,13 +143,13 @@ def test_custom_evaluate_timeout_invalid( ) config_file.close() - with self.assertRaises(ValueError) as err: + with self.assertRaises(ValueError): TabPyApp(self.config_file.name) - @patch("tabpy.tabpy_server.app.app.os") - @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) - @patch("tabpy.tabpy_server.app.app._get_state_from_file") - @patch("tabpy.tabpy_server.app.app.TabPyState") + @patch("tabpy.server.app.app.os") + @patch("tabpy.server.app.app.os.path.exists", return_value=True) + @patch("tabpy.server.app.app._get_state_from_file") + @patch("tabpy.server.app.app.TabPyState") def test_env_variables_in_config( self, mock_state, mock_get_state, mock_path_exists, mock_os ): @@ -234,7 +234,7 @@ def test_https_without_key(self): "Error using HTTPS: The parameter(s) TABPY_KEY_FILE must be set." ) - @patch("tabpy.tabpy_server.app.app.os.path") + @patch("tabpy.server.app.app.os.path") def test_https_cert_and_key_file_not_found(self, mock_path): self.fp.write( "[TabPy]\n" @@ -251,7 +251,7 @@ def test_https_cert_and_key_file_not_found(self, mock_path): "TABPY_KEY_FILE must point to an existing file." ) - @patch("tabpy.tabpy_server.app.app.os.path") + @patch("tabpy.server.app.app.os.path") def test_https_cert_file_not_found(self, mock_path): self.fp.write( "[TabPy]\n" @@ -270,7 +270,7 @@ def test_https_cert_file_not_found(self, mock_path): "must point to an existing file." ) - @patch("tabpy.tabpy_server.app.app.os.path") + @patch("tabpy.server.app.app.os.path") def test_https_key_file_not_found(self, mock_path): self.fp.write( "[TabPy]\n" @@ -289,8 +289,8 @@ def test_https_key_file_not_found(self, mock_path): "must point to an existing file." ) - @patch("tabpy.tabpy_server.app.app.os.path.isfile", return_value=True) - @patch("tabpy.tabpy_server.app.util.validate_cert") + @patch("tabpy.server.app.app.os.path.isfile", return_value=True) + @patch("tabpy.server.app.util.validate_cert") def test_https_success(self, mock_isfile, mock_validate_cert): self.fp.write( "[TabPy]\n" diff --git a/tests/unit/server_tests/test_endpoint_file_manager.py b/tests/unit/server_tests/test_endpoint_file_manager.py index 04ddb9cb..499c18bd 100644 --- a/tests/unit/server_tests/test_endpoint_file_manager.py +++ b/tests/unit/server_tests/test_endpoint_file_manager.py @@ -1,5 +1,5 @@ import unittest -from tabpy.tabpy_server.common.endpoint_file_mgr import _check_endpoint_name +from tabpy.server.common.endpoint_file_mgr import _check_endpoint_name class TestEndpointFileManager(unittest.TestCase): diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index a9393c42..19cede81 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -3,9 +3,9 @@ import sys import tempfile -from tabpy.tabpy_server.app.app import TabPyApp -from tabpy.tabpy_server.app.app import _init_asyncio_patch -from tabpy.tabpy_server.handlers.util import hash_password +from tabpy.server.app.app import TabPyApp +from tabpy.server.app.app import _init_asyncio_patch +from tabpy.server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase diff --git a/tests/unit/server_tests/test_endpoints_handler.py b/tests/unit/server_tests/test_endpoints_handler.py index 6ae00fc7..e3f72aab 100755 --- a/tests/unit/server_tests/test_endpoints_handler.py +++ b/tests/unit/server_tests/test_endpoints_handler.py @@ -2,8 +2,8 @@ import os import tempfile -from tabpy.tabpy_server.app.app import TabPyApp -from tabpy.tabpy_server.handlers.util import hash_password +from tabpy.server.app.app import TabPyApp +from tabpy.server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase diff --git a/tests/unit/server_tests/test_evaluation_plane_handler.py b/tests/unit/server_tests/test_evaluation_plane_handler.py index 49b67dfb..48cce9dc 100755 --- a/tests/unit/server_tests/test_evaluation_plane_handler.py +++ b/tests/unit/server_tests/test_evaluation_plane_handler.py @@ -3,8 +3,8 @@ import tempfile from argparse import Namespace -from tabpy.tabpy_server.app.app import TabPyApp -from tabpy.tabpy_server.handlers.util import hash_password +from tabpy.server.app.app import TabPyApp +from tabpy.server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase diff --git a/tests/unit/server_tests/test_pwd_file.py b/tests/unit/server_tests/test_pwd_file.py index b7c64f96..036b88ac 100755 --- a/tests/unit/server_tests/test_pwd_file.py +++ b/tests/unit/server_tests/test_pwd_file.py @@ -2,7 +2,7 @@ import unittest from tempfile import NamedTemporaryFile -from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.server.app.app import TabPyApp class TestPasswordFile(unittest.TestCase): diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index 4eb82f65..b6a46137 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -1,8 +1,8 @@ import base64 import json import os -from tabpy.tabpy_server.app.app import TabPyApp -from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.server.app.app import TabPyApp +from tabpy.server.app.SettingsParameters import SettingsParameters import tempfile from tornado.testing import AsyncHTTPTestCase @@ -73,11 +73,11 @@ def setUpClass(cls): cls.tabpy_config = ["TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt\n"] super(TestServiceInfoHandlerWithAuth, cls).setUpClass() - def test_given_tabpy_server_with_auth_expect_error_info_response(self): + def test_given_server_with_auth_expect_error_info_response(self): response = self.fetch("/info") self.assertEqual(response.code, 401) - def test_given_tabpy_server_with_auth_expect_correct_info_response(self): + def test_given_server_with_auth_expect_correct_info_response(self): header = { "Content-Type": "application/json", "TabPy-Client": "Integration test for deploying models with auth", @@ -111,7 +111,7 @@ def setUpClass(cls): cls.prefix = "__TestServiceInfoHandlerWithoutAuth_" super(TestServiceInfoHandlerWithoutAuth, cls).setUpClass() - def test_tabpy_server_with_no_auth_expect_correct_info_response(self): + def test_server_with_no_auth_expect_correct_info_response(self): response = self.fetch("/info") self.assertEqual(response.code, 200) actual_response = json.loads(response.body) @@ -128,7 +128,7 @@ def test_tabpy_server_with_no_auth_expect_correct_info_response(self): features = v1["features"] self.assertDictEqual({}, features) - def test_given_tabpy_server_with_no_auth_and_password_expect_correct_info_response(self): + def test_given_server_with_no_auth_and_password_expect_correct_info_response(self): header = { "Content-Type": "application/json", "TabPy-Client": "Integration test for deploying models with auth", diff --git a/tests/unit/tools_tests/test_client.py b/tests/unit/tools_tests/test_client.py index f5df51ff..14416050 100644 --- a/tests/unit/tools_tests/test_client.py +++ b/tests/unit/tools_tests/test_client.py @@ -1,8 +1,8 @@ import unittest from unittest.mock import Mock -from tabpy.tabpy_tools.client import Client -from tabpy.tabpy_tools.client import _check_endpoint_name +from tabpy.tools.client import Client +from tabpy.tools.client import _check_endpoint_name class TestClient(unittest.TestCase): diff --git a/tests/unit/tools_tests/test_rest.py b/tests/unit/tools_tests/test_rest.py index 33a78bf2..445f0dfd 100644 --- a/tests/unit/tools_tests/test_rest.py +++ b/tests/unit/tools_tests/test_rest.py @@ -1,7 +1,7 @@ import json import requests from requests.auth import HTTPBasicAuth -from tabpy.tabpy_tools.rest import RequestsNetworkWrapper, ServiceClient +from tabpy.tools.rest import RequestsNetworkWrapper, ServiceClient import unittest from unittest.mock import Mock diff --git a/tests/unit/tools_tests/test_rest_object.py b/tests/unit/tools_tests/test_rest_object.py index c5fafa17..333c175e 100644 --- a/tests/unit/tools_tests/test_rest_object.py +++ b/tests/unit/tools_tests/test_rest_object.py @@ -1,7 +1,7 @@ import unittest import sys -from tabpy.tabpy_tools.rest import RESTObject, RESTProperty, enum +from tabpy.tools.rest import RESTObject, RESTProperty, enum class TestRESTObject(unittest.TestCase): diff --git a/tests/unit/tools_tests/test_schema.py b/tests/unit/tools_tests/test_schema.py index ba131696..c8b158fd 100755 --- a/tests/unit/tools_tests/test_schema.py +++ b/tests/unit/tools_tests/test_schema.py @@ -1,6 +1,6 @@ import unittest -from tabpy.tabpy_tools.schema import generate_schema +from tabpy.tools.schema import generate_schema class TestSchema(unittest.TestCase): From 68aba488b7d1f9459ca99141b2841a711dfb0223 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Mon, 27 Jul 2020 15:28:25 -0700 Subject: [PATCH 41/61] Remove pypi publishing instructions - those are Tableau specific --- CHANGELOG | 7 +------ CONTRIBUTING.md | 22 ---------------------- tabpy/VERSION | 2 +- 3 files changed, 2 insertions(+), 29 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 8714766d..ad197662 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,16 +1,11 @@ # Changelog -## v2.0.0 +## v1.2.0 ### Improvements - Minor code cleanup. -### Breaking changes - -- tabpy_server and tabpy_tools are renamed to server and tools. - - ## v1.1.0 ### Improvements diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5baaef84..dd5dd55d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -146,25 +146,3 @@ Access-Control-Allow-Methods = GET, OPTIONS, POST ```sh flake8 . ``` - -## Publishing TabPy Package - -Execute the following commands to build and publish a new version of -TabPy package: - -```sh -python setup.py sdist bdist_wheel -python -m twine upload dist/* -``` - -To publish test version of the package use the following command: - -```sh -python -m twine upload --repository-url https://test.pypi.org/legacy/ dist/* -``` - -To install package from TestPyPi use the command: - -```sh -pip install --upgrade -i https://test.pypi.org/simple/ tabpy -``` diff --git a/tabpy/VERSION b/tabpy/VERSION index 227cea21..26aaba0e 100755 --- a/tabpy/VERSION +++ b/tabpy/VERSION @@ -1 +1 @@ -2.0.0 +1.2.0 From d176fbacdfa6a8e5b6647ba13f68914dad0c2c2b Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Mon, 27 Jul 2020 15:57:32 -0700 Subject: [PATCH 42/61] Restore tabpy_tools and tabpy_server names --- CONTRIBUTING.md | 1 - docs/security.md | 2 +- docs/tabpy-tools.md | 6 +- tabpy/models/utils/setup_utils.py | 4 +- tabpy/tabpy.py | 2 +- tabpy/utils/tabpy_user.py | 4 +- tests/unit/server_tests/test_config.py | 66 +++++++++---------- .../test_endpoint_file_manager.py | 2 +- .../server_tests/test_endpoint_handler.py | 6 +- .../server_tests/test_endpoints_handler.py | 4 +- .../test_evaluation_plane_handler.py | 4 +- tests/unit/server_tests/test_pwd_file.py | 2 +- .../server_tests/test_service_info_handler.py | 4 +- tests/unit/tools_tests/test_client.py | 4 +- tests/unit/tools_tests/test_rest.py | 2 +- tests/unit/tools_tests/test_rest_object.py | 2 +- tests/unit/tools_tests/test_schema.py | 2 +- 17 files changed, 58 insertions(+), 59 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dd5dd55d..a8a83541 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,6 @@ - [Documentation Updates](#documentation-updates) - [TabPy with Swagger](#tabpy-with-swagger) - [Code styling](#code-styling) -- [Publishing TabPy Package](#publishing-tabpy-package) diff --git a/docs/security.md b/docs/security.md index 5f8a3c2a..5a0ecb72 100755 --- a/docs/security.md +++ b/docs/security.md @@ -6,7 +6,7 @@ you may want to consider the following as you use TabPy: - The REST server and Python execution share the same Python session, meaning that HTTP requests and user scripts are evaluated in the same addressable memory and processor threads. -- The tabpy.tools client does not perform client-side validation of the +- The tabpy.tabpy_tools client does not perform client-side validation of the SSL certificate on TabPy Server. - Python scripts can contain code which can harm security on the server where the TabPy is running. For example, Python scripts can: diff --git a/docs/tabpy-tools.md b/docs/tabpy-tools.md index a7143156..58a8f31c 100755 --- a/docs/tabpy-tools.md +++ b/docs/tabpy-tools.md @@ -30,7 +30,7 @@ to specify the service location for all subsequent operations: ```python -from tabpy.tools.client import Client +from tabpy.tabpy_tools.client import Client client = Client('http://localhost:9004/') @@ -351,7 +351,7 @@ method provided in this tools package: ```python -from tabpy.tools.schema import generate_schema +from tabpy.tabpy_tools.schema import generate_schema schema = generate_schema( input={'x': 3, 'y': 2}, @@ -368,7 +368,7 @@ To describe more complex input, like arrays, you would use the following syntax: ```python -from tabpy.tools.schema import generate_schema +from tabpy.tabpy_tools.schema import generate_schema schema = generate_schema( input={'x': [6.35, 6.40, 6.65, 8.60], diff --git a/tabpy/models/utils/setup_utils.py b/tabpy/models/utils/setup_utils.py index 5a3f8458..10801fb7 100644 --- a/tabpy/models/utils/setup_utils.py +++ b/tabpy/models/utils/setup_utils.py @@ -2,14 +2,14 @@ import getpass import os import sys -from tabpy.tools.client import Client +from tabpy.tabpy_tools.client import Client def get_default_config_file_path(): import tabpy pkg_path = os.path.dirname(tabpy.__file__) - config_file_path = os.path.join(pkg_path, "server", "common", "default.conf") + config_file_path = os.path.join(pkg_path, "tabpy_server", "common", "default.conf") return config_file_path diff --git a/tabpy/tabpy.py b/tabpy/tabpy.py index bf56c0e1..50acf799 100755 --- a/tabpy/tabpy.py +++ b/tabpy/tabpy.py @@ -38,7 +38,7 @@ def main(): args = docopt.docopt(__doc__) config = args["--config"] or None - from tabpy.server.app.app import TabPyApp + from tabpy.tabpy_server.app.app import TabPyApp app = TabPyApp(config) app.run() diff --git a/tabpy/utils/tabpy_user.py b/tabpy/utils/tabpy_user.py index 0039b84d..065d0298 100755 --- a/tabpy/utils/tabpy_user.py +++ b/tabpy/utils/tabpy_user.py @@ -19,8 +19,8 @@ import docopt import logging import secrets -from tabpy.server.app.util import parse_pwd_file -from tabpy.server.handlers.util import hash_password +from tabpy.tabpy_server.app.util import parse_pwd_file +from tabpy.tabpy_server.handlers.util import hash_password logger = logging.getLogger(__name__) diff --git a/tests/unit/server_tests/test_config.py b/tests/unit/server_tests/test_config.py index 13448334..677c949c 100644 --- a/tests/unit/server_tests/test_config.py +++ b/tests/unit/server_tests/test_config.py @@ -2,8 +2,8 @@ import unittest from tempfile import NamedTemporaryFile import tabpy -from tabpy.server.app.util import validate_cert -from tabpy.server.app.app import TabPyApp +from tabpy.tabpy_server.app.util import validate_cert +from tabpy.tabpy_server.app.app import TabPyApp from unittest.mock import patch @@ -22,11 +22,11 @@ def test_config_file_does_not_exist(self): self.assertEqual(app.settings["log_request_context"], False) self.assertEqual(app.settings["evaluate_timeout"], 30) - @patch("tabpy.server.app.app.TabPyState") - @patch("tabpy.server.app.app._get_state_from_file") - @patch("tabpy.server.app.app.PythonServiceHandler") - @patch("tabpy.server.app.app.os.path.exists", return_value=True) - @patch("tabpy.server.app.app.os") + @patch("tabpy.tabpy_server.app.app.TabPyState") + @patch("tabpy.tabpy_server.app.app._get_state_from_file") + @patch("tabpy.tabpy_server.app.app.PythonServiceHandler") + @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) + @patch("tabpy.tabpy_server.app.app.os") def test_no_config_file( self, mock_os, @@ -37,7 +37,7 @@ def test_no_config_file( ): pkg_path = os.path.dirname(tabpy.__file__) obj_path = os.path.join(pkg_path, "tmp", "query_objects") - state_path = os.path.join(pkg_path, "server") + state_path = os.path.join(pkg_path, "tabpy_server") mock_os.environ = { "TABPY_PORT": "9004", "TABPY_QUERY_OBJECT_PATH": obj_path, @@ -52,11 +52,11 @@ def test_no_config_file( self.assertTrue(len(mock_management_util.mock_calls) > 0) mock_os.makedirs.assert_not_called() - @patch("tabpy.server.app.app.TabPyState") - @patch("tabpy.server.app.app._get_state_from_file") - @patch("tabpy.server.app.app.PythonServiceHandler") - @patch("tabpy.server.app.app.os.path.exists", return_value=False) - @patch("tabpy.server.app.app.os") + @patch("tabpy.tabpy_server.app.app.TabPyState") + @patch("tabpy.tabpy_server.app.app._get_state_from_file") + @patch("tabpy.tabpy_server.app.app.PythonServiceHandler") + @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=False) + @patch("tabpy.tabpy_server.app.app.os") def test_no_state_ini_file_or_state_dir( self, mock_os, @@ -77,11 +77,11 @@ def tearDown(self): os.remove(self.config_file.name) self.config_file = None - @patch("tabpy.server.app.app.TabPyState") - @patch("tabpy.server.app.app._get_state_from_file") - @patch("tabpy.server.app.app.PythonServiceHandler") - @patch("tabpy.server.app.app.os.path.exists", return_value=True) - @patch("tabpy.server.app.app.os") + @patch("tabpy.tabpy_server.app.app.TabPyState") + @patch("tabpy.tabpy_server.app.app._get_state_from_file") + @patch("tabpy.tabpy_server.app.app.PythonServiceHandler") + @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) + @patch("tabpy.tabpy_server.app.app.os") def test_config_file_present( self, mock_os, @@ -116,9 +116,9 @@ def test_config_file_present( self.assertEqual(app.settings["log_request_context"], False) self.assertEqual(app.settings["evaluate_timeout"], 30) - @patch("tabpy.server.app.app.os.path.exists", return_value=True) - @patch("tabpy.server.app.app._get_state_from_file") - @patch("tabpy.server.app.app.TabPyState") + @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) + @patch("tabpy.tabpy_server.app.app._get_state_from_file") + @patch("tabpy.tabpy_server.app.app.TabPyState") def test_custom_evaluate_timeout_valid( self, mock_state, mock_get_state_from_file, mock_path_exists ): @@ -130,9 +130,9 @@ def test_custom_evaluate_timeout_valid( app = TabPyApp(self.config_file.name) self.assertEqual(app.settings["evaluate_timeout"], 1996.0) - @patch("tabpy.server.app.app.os.path.exists", return_value=True) - @patch("tabpy.server.app.app._get_state_from_file") - @patch("tabpy.server.app.app.TabPyState") + @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) + @patch("tabpy.tabpy_server.app.app._get_state_from_file") + @patch("tabpy.tabpy_server.app.app.TabPyState") def test_custom_evaluate_timeout_invalid( self, mock_state, mock_get_state_from_file, mock_path_exists ): @@ -146,10 +146,10 @@ def test_custom_evaluate_timeout_invalid( with self.assertRaises(ValueError): TabPyApp(self.config_file.name) - @patch("tabpy.server.app.app.os") - @patch("tabpy.server.app.app.os.path.exists", return_value=True) - @patch("tabpy.server.app.app._get_state_from_file") - @patch("tabpy.server.app.app.TabPyState") + @patch("tabpy.tabpy_server.app.app.os") + @patch("tabpy.tabpy_server.app.app.os.path.exists", return_value=True) + @patch("tabpy.tabpy_server.app.app._get_state_from_file") + @patch("tabpy.tabpy_server.app.app.TabPyState") def test_env_variables_in_config( self, mock_state, mock_get_state, mock_path_exists, mock_os ): @@ -234,7 +234,7 @@ def test_https_without_key(self): "Error using HTTPS: The parameter(s) TABPY_KEY_FILE must be set." ) - @patch("tabpy.server.app.app.os.path") + @patch("tabpy.tabpy_server.app.app.os.path") def test_https_cert_and_key_file_not_found(self, mock_path): self.fp.write( "[TabPy]\n" @@ -251,7 +251,7 @@ def test_https_cert_and_key_file_not_found(self, mock_path): "TABPY_KEY_FILE must point to an existing file." ) - @patch("tabpy.server.app.app.os.path") + @patch("tabpy.tabpy_server.app.app.os.path") def test_https_cert_file_not_found(self, mock_path): self.fp.write( "[TabPy]\n" @@ -270,7 +270,7 @@ def test_https_cert_file_not_found(self, mock_path): "must point to an existing file." ) - @patch("tabpy.server.app.app.os.path") + @patch("tabpy.tabpy_server.app.app.os.path") def test_https_key_file_not_found(self, mock_path): self.fp.write( "[TabPy]\n" @@ -289,8 +289,8 @@ def test_https_key_file_not_found(self, mock_path): "must point to an existing file." ) - @patch("tabpy.server.app.app.os.path.isfile", return_value=True) - @patch("tabpy.server.app.util.validate_cert") + @patch("tabpy.tabpy_server.app.app.os.path.isfile", return_value=True) + @patch("tabpy.tabpy_server.app.util.validate_cert") def test_https_success(self, mock_isfile, mock_validate_cert): self.fp.write( "[TabPy]\n" diff --git a/tests/unit/server_tests/test_endpoint_file_manager.py b/tests/unit/server_tests/test_endpoint_file_manager.py index 499c18bd..04ddb9cb 100644 --- a/tests/unit/server_tests/test_endpoint_file_manager.py +++ b/tests/unit/server_tests/test_endpoint_file_manager.py @@ -1,5 +1,5 @@ import unittest -from tabpy.server.common.endpoint_file_mgr import _check_endpoint_name +from tabpy.tabpy_server.common.endpoint_file_mgr import _check_endpoint_name class TestEndpointFileManager(unittest.TestCase): diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index 19cede81..a9393c42 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -3,9 +3,9 @@ import sys import tempfile -from tabpy.server.app.app import TabPyApp -from tabpy.server.app.app import _init_asyncio_patch -from tabpy.server.handlers.util import hash_password +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.app.app import _init_asyncio_patch +from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase diff --git a/tests/unit/server_tests/test_endpoints_handler.py b/tests/unit/server_tests/test_endpoints_handler.py index e3f72aab..6ae00fc7 100755 --- a/tests/unit/server_tests/test_endpoints_handler.py +++ b/tests/unit/server_tests/test_endpoints_handler.py @@ -2,8 +2,8 @@ import os import tempfile -from tabpy.server.app.app import TabPyApp -from tabpy.server.handlers.util import hash_password +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase diff --git a/tests/unit/server_tests/test_evaluation_plane_handler.py b/tests/unit/server_tests/test_evaluation_plane_handler.py index 48cce9dc..49b67dfb 100755 --- a/tests/unit/server_tests/test_evaluation_plane_handler.py +++ b/tests/unit/server_tests/test_evaluation_plane_handler.py @@ -3,8 +3,8 @@ import tempfile from argparse import Namespace -from tabpy.server.app.app import TabPyApp -from tabpy.server.handlers.util import hash_password +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.handlers.util import hash_password from tornado.testing import AsyncHTTPTestCase diff --git a/tests/unit/server_tests/test_pwd_file.py b/tests/unit/server_tests/test_pwd_file.py index 036b88ac..b7c64f96 100755 --- a/tests/unit/server_tests/test_pwd_file.py +++ b/tests/unit/server_tests/test_pwd_file.py @@ -2,7 +2,7 @@ import unittest from tempfile import NamedTemporaryFile -from tabpy.server.app.app import TabPyApp +from tabpy.tabpy_server.app.app import TabPyApp class TestPasswordFile(unittest.TestCase): diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index b6a46137..767a50c0 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -1,8 +1,8 @@ import base64 import json import os -from tabpy.server.app.app import TabPyApp -from tabpy.server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.app.app import TabPyApp +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters import tempfile from tornado.testing import AsyncHTTPTestCase diff --git a/tests/unit/tools_tests/test_client.py b/tests/unit/tools_tests/test_client.py index 14416050..f5df51ff 100644 --- a/tests/unit/tools_tests/test_client.py +++ b/tests/unit/tools_tests/test_client.py @@ -1,8 +1,8 @@ import unittest from unittest.mock import Mock -from tabpy.tools.client import Client -from tabpy.tools.client import _check_endpoint_name +from tabpy.tabpy_tools.client import Client +from tabpy.tabpy_tools.client import _check_endpoint_name class TestClient(unittest.TestCase): diff --git a/tests/unit/tools_tests/test_rest.py b/tests/unit/tools_tests/test_rest.py index 445f0dfd..33a78bf2 100644 --- a/tests/unit/tools_tests/test_rest.py +++ b/tests/unit/tools_tests/test_rest.py @@ -1,7 +1,7 @@ import json import requests from requests.auth import HTTPBasicAuth -from tabpy.tools.rest import RequestsNetworkWrapper, ServiceClient +from tabpy.tabpy_tools.rest import RequestsNetworkWrapper, ServiceClient import unittest from unittest.mock import Mock diff --git a/tests/unit/tools_tests/test_rest_object.py b/tests/unit/tools_tests/test_rest_object.py index 333c175e..c5fafa17 100644 --- a/tests/unit/tools_tests/test_rest_object.py +++ b/tests/unit/tools_tests/test_rest_object.py @@ -1,7 +1,7 @@ import unittest import sys -from tabpy.tools.rest import RESTObject, RESTProperty, enum +from tabpy.tabpy_tools.rest import RESTObject, RESTProperty, enum class TestRESTObject(unittest.TestCase): diff --git a/tests/unit/tools_tests/test_schema.py b/tests/unit/tools_tests/test_schema.py index c8b158fd..ba131696 100755 --- a/tests/unit/tools_tests/test_schema.py +++ b/tests/unit/tools_tests/test_schema.py @@ -1,6 +1,6 @@ import unittest -from tabpy.tools.schema import generate_schema +from tabpy.tabpy_tools.schema import generate_schema class TestSchema(unittest.TestCase): From b51d0325b56443a1d1420fdf14f25c20e10c54a6 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Mon, 27 Jul 2020 15:59:51 -0700 Subject: [PATCH 43/61] Restore tabpy_tools and tabpy_server names --- tabpy/server/__init__.py | 0 tabpy/server/app/ConfigParameters.py | 17 - tabpy/server/app/SettingsParameters.py | 17 - tabpy/server/app/__init__.py | 0 tabpy/server/app/app.py | 430 ------------ tabpy/server/app/util.py | 88 --- tabpy/server/common/__init__.py | 0 tabpy/server/common/default.conf | 62 -- tabpy/server/common/endpoint_file_mgr.py | 94 --- tabpy/server/common/messages.py | 172 ----- tabpy/server/common/util.py | 3 - tabpy/server/handlers/__init__.py | 13 - tabpy/server/handlers/base_handler.py | 425 ------------ tabpy/server/handlers/endpoint_handler.py | 141 ---- tabpy/server/handlers/endpoints_handler.py | 75 -- .../handlers/evaluation_plane_handler.py | 142 ---- tabpy/server/handlers/main_handler.py | 7 - tabpy/server/handlers/management_handler.py | 150 ---- tabpy/server/handlers/query_plane_handler.py | 233 ------- tabpy/server/handlers/service_info_handler.py | 23 - tabpy/server/handlers/status_handler.py | 29 - .../handlers/upload_destination_handler.py | 20 - tabpy/server/handlers/util.py | 32 - tabpy/server/management/__init__.py | 0 tabpy/server/management/state.py | 643 ------------------ tabpy/server/management/util.py | 46 -- tabpy/server/psws/__init__.py | 0 tabpy/server/psws/callbacks.py | 205 ------ tabpy/server/psws/python_service.py | 275 -------- tabpy/server/state.ini.template | 15 - tabpy/server/static/index.html | 72 -- tabpy/server/static/tableau.png | Bin 33575 -> 0 bytes tabpy/tools/__init__.py | 0 tabpy/tools/client.py | 389 ----------- tabpy/tools/custom_query_object.py | 83 --- tabpy/tools/query_object.py | 108 --- tabpy/tools/rest.py | 423 ------------ tabpy/tools/rest_client.py | 252 ------- tabpy/tools/schema.py | 108 --- 39 files changed, 4792 deletions(-) delete mode 100644 tabpy/server/__init__.py delete mode 100644 tabpy/server/app/ConfigParameters.py delete mode 100644 tabpy/server/app/SettingsParameters.py delete mode 100644 tabpy/server/app/__init__.py delete mode 100644 tabpy/server/app/app.py delete mode 100644 tabpy/server/app/util.py delete mode 100644 tabpy/server/common/__init__.py delete mode 100644 tabpy/server/common/default.conf delete mode 100644 tabpy/server/common/endpoint_file_mgr.py delete mode 100644 tabpy/server/common/messages.py delete mode 100644 tabpy/server/common/util.py delete mode 100644 tabpy/server/handlers/__init__.py delete mode 100644 tabpy/server/handlers/base_handler.py delete mode 100644 tabpy/server/handlers/endpoint_handler.py delete mode 100644 tabpy/server/handlers/endpoints_handler.py delete mode 100644 tabpy/server/handlers/evaluation_plane_handler.py delete mode 100644 tabpy/server/handlers/main_handler.py delete mode 100644 tabpy/server/handlers/management_handler.py delete mode 100644 tabpy/server/handlers/query_plane_handler.py delete mode 100644 tabpy/server/handlers/service_info_handler.py delete mode 100644 tabpy/server/handlers/status_handler.py delete mode 100644 tabpy/server/handlers/upload_destination_handler.py delete mode 100644 tabpy/server/handlers/util.py delete mode 100644 tabpy/server/management/__init__.py delete mode 100644 tabpy/server/management/state.py delete mode 100644 tabpy/server/management/util.py delete mode 100644 tabpy/server/psws/__init__.py delete mode 100644 tabpy/server/psws/callbacks.py delete mode 100644 tabpy/server/psws/python_service.py delete mode 100644 tabpy/server/state.ini.template delete mode 100644 tabpy/server/static/index.html delete mode 100644 tabpy/server/static/tableau.png delete mode 100644 tabpy/tools/__init__.py delete mode 100644 tabpy/tools/client.py delete mode 100644 tabpy/tools/custom_query_object.py delete mode 100644 tabpy/tools/query_object.py delete mode 100644 tabpy/tools/rest.py delete mode 100644 tabpy/tools/rest_client.py delete mode 100644 tabpy/tools/schema.py diff --git a/tabpy/server/__init__.py b/tabpy/server/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tabpy/server/app/ConfigParameters.py b/tabpy/server/app/ConfigParameters.py deleted file mode 100644 index 3feca9b7..00000000 --- a/tabpy/server/app/ConfigParameters.py +++ /dev/null @@ -1,17 +0,0 @@ -class ConfigParameters: - """ - Configuration settings names - """ - - TABPY_PWD_FILE = "TABPY_PWD_FILE" - TABPY_PORT = "TABPY_PORT" - TABPY_QUERY_OBJECT_PATH = "TABPY_QUERY_OBJECT_PATH" - TABPY_STATE_PATH = "TABPY_STATE_PATH" - TABPY_TRANSFER_PROTOCOL = "TABPY_TRANSFER_PROTOCOL" - TABPY_CERTIFICATE_FILE = "TABPY_CERTIFICATE_FILE" - TABPY_KEY_FILE = "TABPY_KEY_FILE" - TABPY_PWD_FILE = "TABPY_PWD_FILE" - TABPY_LOG_DETAILS = "TABPY_LOG_DETAILS" - TABPY_STATIC_PATH = "TABPY_STATIC_PATH" - TABPY_MAX_REQUEST_SIZE_MB = "TABPY_MAX_REQUEST_SIZE_MB" - TABPY_EVALUATE_TIMEOUT = "TABPY_EVALUATE_TIMEOUT" diff --git a/tabpy/server/app/SettingsParameters.py b/tabpy/server/app/SettingsParameters.py deleted file mode 100644 index 45fb128a..00000000 --- a/tabpy/server/app/SettingsParameters.py +++ /dev/null @@ -1,17 +0,0 @@ -class SettingsParameters: - """ - Application (TabPyApp) settings names - """ - - TransferProtocol = "transfer_protocol" - Port = "port" - ServerVersion = "server_version" - UploadDir = "upload_dir" - CertificateFile = "certificate_file" - KeyFile = "key_file" - StateFilePath = "state_file_path" - ApiVersions = "versions" - LogRequestContext = "log_request_context" - StaticPath = "static_path" - MaxRequestSizeInMb = "max_request_size_in_mb" - EvaluateTimeout = "evaluate_timeout" diff --git a/tabpy/server/app/__init__.py b/tabpy/server/app/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tabpy/server/app/app.py b/tabpy/server/app/app.py deleted file mode 100644 index 4864b742..00000000 --- a/tabpy/server/app/app.py +++ /dev/null @@ -1,430 +0,0 @@ -import concurrent.futures -import configparser -import logging -import multiprocessing -import os -import shutil -import signal -import sys -import tabpy -from tabpy.tabpy import __version__ -from tabpy.server.app.ConfigParameters import ConfigParameters -from tabpy.server.app.SettingsParameters import SettingsParameters -from tabpy.server.app.util import parse_pwd_file -from tabpy.server.management.state import TabPyState -from tabpy.server.management.util import _get_state_from_file -from tabpy.server.psws.callbacks import init_model_evaluator, init_ps_server -from tabpy.server.psws.python_service import PythonService, PythonServiceHandler -from tabpy.server.handlers import ( - EndpointHandler, - EndpointsHandler, - EvaluationPlaneHandler, - QueryPlaneHandler, - ServiceInfoHandler, - StatusHandler, - UploadDestinationHandler, -) -import tornado - - -logger = logging.getLogger(__name__) - - -def _init_asyncio_patch(): - """ - Select compatible event loop for Tornado 5+. - As of Python 3.8, the default event loop on Windows is `proactor`, - however Tornado requires the old default "selector" event loop. - As Tornado has decided to leave this to users to set, MkDocs needs - to set it. See https://github.com/tornadoweb/tornado/issues/2608. - """ - if sys.platform.startswith("win") and sys.version_info >= (3, 8): - import asyncio - try: - from asyncio import WindowsSelectorEventLoopPolicy - except ImportError: - pass # Can't assign a policy which doesn't exist. - else: - if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy): - asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) - - -class TabPyApp: - """ - TabPy application class for keeping context like settings, state, etc. - """ - - settings = {} - subdirectory = "" - tabpy_state = None - python_service = None - credentials = {} - - def __init__(self, config_file=None): - if config_file is None: - config_file = os.path.join( - os.path.dirname(__file__), os.path.pardir, "common", "default.conf" - ) - - if os.path.isfile(config_file): - try: - from logging import config - config.fileConfig(config_file, disable_existing_loggers=False) - except KeyError: - logging.basicConfig(level=logging.DEBUG) - - self._parse_config(config_file) - - def run(self): - application = self._create_tornado_web_app() - max_request_size = ( - int(self.settings[SettingsParameters.MaxRequestSizeInMb]) * 1024 * 1024 - ) - logger.info(f"Setting max request size to {max_request_size} bytes") - - init_model_evaluator(self.settings, self.tabpy_state, self.python_service) - - protocol = self.settings[SettingsParameters.TransferProtocol] - ssl_options = None - if protocol == "https": - ssl_options = { - "certfile": self.settings[SettingsParameters.CertificateFile], - "keyfile": self.settings[SettingsParameters.KeyFile], - } - elif protocol != "http": - msg = f"Unsupported transfer protocol {protocol}." - logger.critical(msg) - raise RuntimeError(msg) - - application.listen( - self.settings[SettingsParameters.Port], - ssl_options=ssl_options, - max_buffer_size=max_request_size, - max_body_size=max_request_size, - ) - - logger.info( - "Web service listening on port " - f"{str(self.settings[SettingsParameters.Port])}" - ) - tornado.ioloop.IOLoop.instance().start() - - def _create_tornado_web_app(self): - class TabPyTornadoApp(tornado.web.Application): - is_closing = False - - def signal_handler(self, signal, _): - logger.critical(f"Exiting on signal {signal}...") - self.is_closing = True - - def try_exit(self): - if self.is_closing: - tornado.ioloop.IOLoop.instance().stop() - logger.info("Shutting down TabPy...") - - logger.info("Initializing TabPy...") - tornado.ioloop.IOLoop.instance().run_sync( - lambda: init_ps_server(self.settings, self.tabpy_state) - ) - logger.info("Done initializing TabPy.") - - executor = concurrent.futures.ThreadPoolExecutor( - max_workers=multiprocessing.cpu_count() - ) - - # initialize Tornado application - _init_asyncio_patch() - application = TabPyTornadoApp( - [ - # skip MainHandler to use StaticFileHandler .* page requests and - # default to index.html - # (r"/", MainHandler), - ( - self.subdirectory + r"/query/([^/]+)", - QueryPlaneHandler, - dict(app=self), - ), - (self.subdirectory + r"/status", StatusHandler, dict(app=self)), - (self.subdirectory + r"/info", ServiceInfoHandler, dict(app=self)), - (self.subdirectory + r"/endpoints", EndpointsHandler, dict(app=self)), - ( - self.subdirectory + r"/endpoints/([^/]+)?", - EndpointHandler, - dict(app=self), - ), - ( - self.subdirectory + r"/evaluate", - EvaluationPlaneHandler, - dict(executor=executor, app=self), - ), - ( - self.subdirectory + r"/configurations/endpoint_upload_destination", - UploadDestinationHandler, - dict(app=self), - ), - ( - self.subdirectory + r"/(.*)", - tornado.web.StaticFileHandler, - dict( - path=self.settings[SettingsParameters.StaticPath], - default_filename="index.html", - ), - ), - ], - debug=False, - **self.settings, - ) - - signal.signal(signal.SIGINT, application.signal_handler) - tornado.ioloop.PeriodicCallback(application.try_exit, 500).start() - - signal.signal(signal.SIGINT, application.signal_handler) - tornado.ioloop.PeriodicCallback(application.try_exit, 500).start() - - return application - - def _set_parameter(self, parser, settings_key, config_key, default_val, parse_function): - key_is_set = False - - if ( - config_key is not None - and parser.has_section("TabPy") - and parser.has_option("TabPy", config_key) - ): - if parse_function is None: - parse_function = parser.get - self.settings[settings_key] = parse_function("TabPy", config_key) - key_is_set = True - logger.debug( - f"Parameter {settings_key} set to " - f'"{self.settings[settings_key]}" ' - "from config file or environment variable" - ) - - if not key_is_set and default_val is not None: - self.settings[settings_key] = default_val - key_is_set = True - logger.debug( - f"Parameter {settings_key} set to " - f'"{self.settings[settings_key]}" ' - "from default value" - ) - - if not key_is_set: - logger.debug(f"Parameter {settings_key} is not set") - - def _parse_config(self, config_file): - """Provide consistent mechanism for pulling in configuration. - - Attempt to retain backward compatibility for - existing implementations by grabbing port - setting from CLI first. - - Take settings in the following order: - - 1. CLI arguments if present - 2. config file - 3. OS environment variables (for ease of - setting defaults if not present) - 4. current defaults if a setting is not present in any location - - Additionally provide similar configuration capabilities in between - config file and environment variables. - For consistency use the same variable name in the config file as - in the os environment. - For naming standards use all capitals and start with 'TABPY_' - """ - self.settings = {} - self.subdirectory = "" - self.tabpy_state = None - self.python_service = None - self.credentials = {} - - pkg_path = os.path.dirname(tabpy.__file__) - - parser = configparser.ConfigParser(os.environ) - - if os.path.isfile(config_file): - with open(config_file) as f: - parser.read_string(f.read()) - else: - logger.warning( - f"Unable to find config file at {config_file}, " - "using default settings." - ) - - settings_parameters = [ - (SettingsParameters.Port, ConfigParameters.TABPY_PORT, 9004, None), - (SettingsParameters.ServerVersion, None, __version__, None), - (SettingsParameters.EvaluateTimeout, ConfigParameters.TABPY_EVALUATE_TIMEOUT, - 30, parser.getfloat), - (SettingsParameters.UploadDir, ConfigParameters.TABPY_QUERY_OBJECT_PATH, - os.path.join(pkg_path, "tmp", "query_objects"), None), - (SettingsParameters.TransferProtocol, ConfigParameters.TABPY_TRANSFER_PROTOCOL, - "http", None), - (SettingsParameters.CertificateFile, ConfigParameters.TABPY_CERTIFICATE_FILE, - None, None), - (SettingsParameters.KeyFile, ConfigParameters.TABPY_KEY_FILE, None, None), - (SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, - os.path.join(pkg_path, "server"), None), - (SettingsParameters.StaticPath, ConfigParameters.TABPY_STATIC_PATH, - os.path.join(pkg_path, "server", "static"), None), - (ConfigParameters.TABPY_PWD_FILE, ConfigParameters.TABPY_PWD_FILE, None, None), - (SettingsParameters.LogRequestContext, ConfigParameters.TABPY_LOG_DETAILS, - "false", None), - (SettingsParameters.MaxRequestSizeInMb, ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB, - 100, None), - ] - - for setting, parameter, default_val, parse_function in settings_parameters: - self._set_parameter(parser, setting, parameter, default_val, parse_function) - - if not os.path.exists(self.settings[SettingsParameters.UploadDir]): - os.makedirs(self.settings[SettingsParameters.UploadDir]) - - # set and validate transfer protocol - self.settings[SettingsParameters.TransferProtocol] = self.settings[ - SettingsParameters.TransferProtocol - ].lower() - - self._validate_transfer_protocol_settings() - - # if state.ini does not exist try and create it - remove - # last dependence on batch/shell script - self.settings[SettingsParameters.StateFilePath] = os.path.realpath( - os.path.normpath( - os.path.expanduser(self.settings[SettingsParameters.StateFilePath]) - ) - ) - state_config, self.tabpy_state = self._build_tabpy_state() - - self.python_service = PythonServiceHandler(PythonService()) - self.settings["compress_response"] = True - self.settings[SettingsParameters.StaticPath] = os.path.abspath( - self.settings[SettingsParameters.StaticPath] - ) - logger.debug( - f"Static pages folder set to " - f'"{self.settings[SettingsParameters.StaticPath]}"' - ) - - # Set subdirectory from config if applicable - if state_config.has_option("Service Info", "Subdirectory"): - self.subdirectory = "/" + state_config.get("Service Info", "Subdirectory") - - # If passwords file specified load credentials - if ConfigParameters.TABPY_PWD_FILE in self.settings: - if not self._parse_pwd_file(): - msg = ( - "Failed to read passwords file " - f"{self.settings[ConfigParameters.TABPY_PWD_FILE]}" - ) - logger.critical(msg) - raise RuntimeError(msg) - else: - logger.info( - "Password file is not specified: " "Authentication is not enabled" - ) - - features = self._get_features() - self.settings[SettingsParameters.ApiVersions] = {"v1": {"features": features}} - - self.settings[SettingsParameters.LogRequestContext] = ( - self.settings[SettingsParameters.LogRequestContext].lower() != "false" - ) - call_context_state = ( - "enabled" - if self.settings[SettingsParameters.LogRequestContext] - else "disabled" - ) - logger.info(f"Call context logging is {call_context_state}") - - def _validate_transfer_protocol_settings(self): - if SettingsParameters.TransferProtocol not in self.settings: - msg = "Missing transfer protocol information." - logger.critical(msg) - raise RuntimeError(msg) - - protocol = self.settings[SettingsParameters.TransferProtocol] - - if protocol == "http": - return - - if protocol != "https": - msg = f"Unsupported transfer protocol: {protocol}" - logger.critical(msg) - raise RuntimeError(msg) - - self._validate_cert_key_state( - "The parameter(s) {} must be set.", - SettingsParameters.CertificateFile in self.settings, - SettingsParameters.KeyFile in self.settings, - ) - cert = self.settings[SettingsParameters.CertificateFile] - - self._validate_cert_key_state( - "The parameter(s) {} must point to " "an existing file.", - os.path.isfile(cert), - os.path.isfile(self.settings[SettingsParameters.KeyFile]), - ) - tabpy.server.app.util.validate_cert(cert) - - @staticmethod - def _validate_cert_key_state(msg, cert_valid, key_valid): - cert_and_key_param = ( - f"{ConfigParameters.TABPY_CERTIFICATE_FILE} and " - f"{ConfigParameters.TABPY_KEY_FILE}" - ) - https_error = "Error using HTTPS: " - err = None - if not cert_valid and not key_valid: - err = https_error + msg.format(cert_and_key_param) - elif not cert_valid: - err = https_error + msg.format(ConfigParameters.TABPY_CERTIFICATE_FILE) - elif not key_valid: - err = https_error + msg.format(ConfigParameters.TABPY_KEY_FILE) - - if err is not None: - logger.critical(err) - raise RuntimeError(err) - - def _parse_pwd_file(self): - succeeded, self.credentials = parse_pwd_file( - self.settings[ConfigParameters.TABPY_PWD_FILE] - ) - - if succeeded and len(self.credentials) == 0: - logger.error("No credentials found") - succeeded = False - - return succeeded - - def _get_features(self): - features = {} - - # Check for auth - if ConfigParameters.TABPY_PWD_FILE in self.settings: - features["authentication"] = { - "required": True, - "methods": {"basic-auth": {}}, - } - - return features - - def _build_tabpy_state(self): - pkg_path = os.path.dirname(tabpy.__file__) - state_file_dir = self.settings[SettingsParameters.StateFilePath] - state_file_path = os.path.join(state_file_dir, "state.ini") - if not os.path.isfile(state_file_path): - state_file_template_path = os.path.join( - pkg_path, "server", "state.ini.template" - ) - logger.debug( - f"File {state_file_path} not found, creating from " - f"template {state_file_template_path}..." - ) - shutil.copy(state_file_template_path, state_file_path) - - logger.info(f"Loading state from state file {state_file_path}") - tabpy_state = _get_state_from_file(state_file_dir) - return tabpy_state, TabPyState(config=tabpy_state, settings=self.settings) diff --git a/tabpy/server/app/util.py b/tabpy/server/app/util.py deleted file mode 100644 index 944b5997..00000000 --- a/tabpy/server/app/util.py +++ /dev/null @@ -1,88 +0,0 @@ -import csv -from datetime import datetime -import logging -from OpenSSL import crypto -import os - - -logger = logging.getLogger(__name__) - - -def validate_cert(cert_file_path): - with open(cert_file_path, "r") as f: - cert_buf = f.read() - - cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_buf) - - date_format, encoding = "%Y%m%d%H%M%SZ", "ascii" - not_before = datetime.strptime(cert.get_notBefore().decode(encoding), date_format) - not_after = datetime.strptime(cert.get_notAfter().decode(encoding), date_format) - now = datetime.now() - - https_error = "Error using HTTPS: " - if now < not_before: - msg = https_error + f"The certificate provided is not valid until {not_before}." - logger.critical(msg) - raise RuntimeError(msg) - if now > not_after: - msg = https_error + f"The certificate provided expired on {not_after}." - logger.critical(msg) - raise RuntimeError(msg) - - -def parse_pwd_file(pwd_file_name): - """ - Parses passwords file and returns set of credentials. - - Parameters - ---------- - pwd_file_name : str - Passwords file name. - - Returns - ------- - succeeded : bool - True if specified file was parsed successfully. - False if there were any issues with parsing specified file. - - credentials : dict - Credentials from the file. Empty if succeeded is False. - """ - logger.info(f"Parsing passwords file {pwd_file_name}...") - - if not os.path.isfile(pwd_file_name): - logger.critical(f"Passwords file {pwd_file_name} not found") - return False, {} - - credentials = {} - with open(pwd_file_name) as pwd_file: - pwd_file_reader = csv.reader(pwd_file, delimiter=" ") - for row in pwd_file_reader: - # skip empty lines - if len(row) == 0: - continue - - # skip commented lines - if row[0][0] == "#": - continue - - if len(row) != 2: - logger.error(f'Incorrect entry "{row}" in password file') - return False, {} - - login = row[0].lower() - if login in credentials: - logger.error( - f"Multiple entries for username {login} in password file" - ) - return False, {} - - if len(row[1]) > 0: - credentials[login] = row[1] - logger.debug(f"Found username {login}") - else: - logger.warning(f"Found username {row[0]} but no password") - return False, {} - - logger.info("Authentication is enabled") - return True, credentials diff --git a/tabpy/server/common/__init__.py b/tabpy/server/common/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tabpy/server/common/default.conf b/tabpy/server/common/default.conf deleted file mode 100644 index 80e9dd49..00000000 --- a/tabpy/server/common/default.conf +++ /dev/null @@ -1,62 +0,0 @@ -[TabPy] -# TABPY_QUERY_OBJECT_PATH = /tmp/query_objects -# TABPY_PORT = 9004 -# TABPY_STATE_PATH = ./tabpy/server - -# Where static pages live -# TABPY_STATIC_PATH = ./tabpy/server/static - -# For how to configure TabPy authentication read -# Authentication section in docs/server-config.md. -# TABPY_PWD_FILE = /path/to/password/file.txt - -# To set up secure TabPy uncomment and modify the following lines. -# Note only PEM-encoded x509 certificates are supported. -# TABPY_TRANSFER_PROTOCOL = https -# TABPY_CERTIFICATE_FILE = /path/to/certificate/file.crt -# TABPY_KEY_FILE = /path/to/key/file.key - -# Log additional request details including caller IP, full URL, client -# end user info if provided. -# TABPY_LOG_DETAILS = true - -# Limit request size (in Mb) - any request which size exceeds -# specified amount will be rejected by TabPy. -# Default value is 100 Mb. -# TABPY_MAX_REQUEST_SIZE_MB = 100 - -# Configure how long a custom script provided to the /evaluate method -# will run before throwing a TimeoutError. -# The value should be a float representing the timeout time in seconds. -# TABPY_EVALUATE_TIMEOUT = 30 - -[loggers] -keys=root - -[handlers] -keys=rootHandler,rotatingFileHandler - -[formatters] -keys=rootFormatter - -[logger_root] -level=DEBUG -handlers=rootHandler,rotatingFileHandler -qualname=root -propagete=0 - -[handler_rootHandler] -class=StreamHandler -level=DEBUG -formatter=rootFormatter -args=(sys.stdout,) - -[handler_rotatingFileHandler] -class=handlers.RotatingFileHandler -level=DEBUG -formatter=rootFormatter -args=('tabpy_log.log', 'a', 1000000, 5) - -[formatter_rootFormatter] -format=%(asctime)s [%(levelname)s] (%(filename)s:%(module)s:%(lineno)d): %(message)s -datefmt=%Y-%m-%d,%H:%M:%S diff --git a/tabpy/server/common/endpoint_file_mgr.py b/tabpy/server/common/endpoint_file_mgr.py deleted file mode 100644 index 6b7fed00..00000000 --- a/tabpy/server/common/endpoint_file_mgr.py +++ /dev/null @@ -1,94 +0,0 @@ -""" -This module provides functionality required for managing endpoint objects in -TabPy. It provides a way to download endpoint files from remote -and then properly cleanup local the endpoint files on update/remove of endpoint -objects. - -The local temporary files for TabPy will by default located at - /tmp/query_objects - -""" -import logging -import os -import shutil -from re import compile as _compile - - -_name_checker = _compile(r"^[a-zA-Z0-9-_\s]+$") - - -def _check_endpoint_name(name, logger=logging.getLogger(__name__)): - """Checks that the endpoint name is valid by comparing it with an RE and - checking that it is not reserved.""" - if not isinstance(name, str): - msg = "Endpoint name must be a string" - logger.log(logging.CRITICAL, msg) - raise TypeError(msg) - - if name == "": - msg = "Endpoint name cannot be empty" - logger.log(logging.CRITICAL, msg) - raise ValueError(msg) - - if not _name_checker.match(name): - msg = ( - "Endpoint name can only contain: a-z, A-Z, 0-9," - " underscore, hyphens and spaces." - ) - logger.log(logging.CRITICAL, msg) - raise ValueError(msg) - - -def grab_files(directory): - """ - Generator that returns all files in a directory. - """ - if not os.path.isdir(directory): - return - else: - for name in os.listdir(directory): - full_path = os.path.join(directory, name) - if os.path.isdir(full_path): - for entry in grab_files(full_path): - yield entry - elif os.path.isfile(full_path): - yield full_path - - -def cleanup_endpoint_files( - name, query_path, logger=logging.getLogger(__name__), retain_versions=None -): - """ - Cleanup the disk space a certain endpiont uses. - - Parameters - ---------- - name : str - The endpoint name - - retain_version : int, optional - If given, then all files for this endpoint are removed except the - folder for the given version, otherwise, all files for that endpoint - are removed. - """ - _check_endpoint_name(name, logger=logger) - local_dir = os.path.join(query_path, name) - - # nothing to clean, this is true for state file path where we load - # Query Object directly from the state path instead of downloading - # to temporary location - if not os.path.exists(local_dir): - return - - if not retain_versions: - shutil.rmtree(local_dir) - else: - retain_folders = [ - os.path.join(local_dir, str(version)) for version in retain_versions - ] - logger.log(logging.INFO, f"Retain folders: {retain_folders}") - - for file_or_dir in os.listdir(local_dir): - candidate_dir = os.path.join(local_dir, file_or_dir) - if os.path.isdir(candidate_dir) and (candidate_dir not in retain_folders): - shutil.rmtree(candidate_dir) diff --git a/tabpy/server/common/messages.py b/tabpy/server/common/messages.py deleted file mode 100644 index ad684319..00000000 --- a/tabpy/server/common/messages.py +++ /dev/null @@ -1,172 +0,0 @@ -import abc -from abc import ABCMeta -from collections import namedtuple -import json - - -class Msg: - """ - An abstract base class for all messages used for communicating between - the WebServices. - - The minimal functionality is the ability to instantiate a Msg from JSON - and to write a Msg instance to JSON. - - We use namedtuples because they are lightweight and immutable. The splat - operator (*) that we inherit from namedtuple is also convenient. We empty - __slots__ to avoid unnecessary overhead. - """ - - __metaclass__ = ABCMeta - - @abc.abstractmethod - def for_json(self): - d = self._asdict() - type_str = self.__class__.__name__ - d.update({"type": type_str}) - return d - - @abc.abstractmethod - def to_json(self): - return json.dumps(self.for_json()) - - @staticmethod - def from_json(str): - d = json.loads(str) - type_str = d["type"] - del d["type"] - return eval(type_str)(**d) - - -class LoadSuccessful( - namedtuple( - "LoadSuccessful", ["uri", "path", "version", "is_update", "endpoint_type"] - ), - Msg, -): - __slots__ = () - - -class LoadFailed(namedtuple("LoadFailed", ["uri", "version", "error_msg"]), Msg): - __slots__ = () - - -class LoadInProgress( - namedtuple( - "LoadInProgress", ["uri", "path", "version", "is_update", "endpoint_type"] - ), - Msg, -): - __slots__ = () - - -class Query(namedtuple("Query", ["uri", "params"]), Msg): - __slots__ = () - - -class QuerySuccessful( - namedtuple("QuerySuccessful", ["uri", "version", "response"]), Msg -): - __slots__ = () - - -class LoadObject( - namedtuple("LoadObject", ["uri", "url", "version", "is_update", "endpoint_type"]), - Msg, -): - __slots__ = () - - -class DeleteObjects(namedtuple("DeleteObjects", ["uris"]), Msg): - __slots__ = () - - -# Used for testing to flush out objects -class FlushObjects(namedtuple("FlushObjects", []), Msg): - __slots__ = () - - -class ObjectsDeleted(namedtuple("ObjectsDeleted", ["uris"]), Msg): - __slots__ = () - - -class ObjectsFlushed(namedtuple("ObjectsFlushed", ["n_before", "n_after"]), Msg): - __slots__ = () - - -class CountObjects(namedtuple("CountObjects", []), Msg): - __slots__ = () - - -class ObjectCount(namedtuple("ObjectCount", ["count"]), Msg): - __slots__ = () - - -class ListObjects(namedtuple("ListObjects", []), Msg): - __slots__ = () - - -class ObjectList(namedtuple("ObjectList", ["objects"]), Msg): - __slots__ = () - - -class UnknownURI(namedtuple("UnknownURI", ["uri"]), Msg): - __slots__ = () - - -class UnknownMessage(namedtuple("UnknownMessage", ["msg"]), Msg): - __slots__ = () - - -class DownloadSkipped( - namedtuple("DownloadSkipped", ["uri", "version", "msg", "host"]), Msg -): - __slots__ = () - - -class QueryFailed(namedtuple("QueryFailed", ["uri", "error"]), Msg): - __slots__ = () - - -class QueryError(namedtuple("QueryError", ["uri", "error"]), Msg): - __slots__ = () - - -class CheckHealth(namedtuple("CheckHealth", []), Msg): - __slots__ = () - - -class Healthy(namedtuple("Healthy", []), Msg): - __slots__ = () - - -class Unhealthy(namedtuple("Unhealthy", []), Msg): - __slots__ = () - - -class Ping(namedtuple("Ping", ["id"]), Msg): - __slots__ = () - - -class Pong(namedtuple("Pong", ["id"]), Msg): - __slots__ = () - - -class Listening(namedtuple("Listening", []), Msg): - __slots__ = () - - -class EngineFailure(namedtuple("EngineFailure", ["error"]), Msg): - __slots__ = () - - -class FlushLogs(namedtuple("FlushLogs", []), Msg): - __slots__ = () - - -class LogsFlushed(namedtuple("LogsFlushed", []), Msg): - __slots__ = () - - -class ServiceError(namedtuple("ServiceError", ["error"]), Msg): - __slots__ = () diff --git a/tabpy/server/common/util.py b/tabpy/server/common/util.py deleted file mode 100644 index c731450a..00000000 --- a/tabpy/server/common/util.py +++ /dev/null @@ -1,3 +0,0 @@ -def format_exception(e, context): - err_msg = f"{e.__class__.__name__} : {str(e)}" - return err_msg diff --git a/tabpy/server/handlers/__init__.py b/tabpy/server/handlers/__init__.py deleted file mode 100644 index fbf68bd7..00000000 --- a/tabpy/server/handlers/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from tabpy.server.handlers.base_handler import BaseHandler -from tabpy.server.handlers.main_handler import MainHandler -from tabpy.server.handlers.management_handler import ManagementHandler - -from tabpy.server.handlers.endpoint_handler import EndpointHandler -from tabpy.server.handlers.endpoints_handler import EndpointsHandler -from tabpy.server.handlers.evaluation_plane_handler import EvaluationPlaneHandler -from tabpy.server.handlers.query_plane_handler import QueryPlaneHandler -from tabpy.server.handlers.service_info_handler import ServiceInfoHandler -from tabpy.server.handlers.status_handler import StatusHandler -from tabpy.server.handlers.upload_destination_handler import ( - UploadDestinationHandler, -) diff --git a/tabpy/server/handlers/base_handler.py b/tabpy/server/handlers/base_handler.py deleted file mode 100644 index 853a00a5..00000000 --- a/tabpy/server/handlers/base_handler.py +++ /dev/null @@ -1,425 +0,0 @@ -import base64 -import binascii -import concurrent -import json -import logging -import tornado.web -from tabpy.server.app.SettingsParameters import SettingsParameters -from tabpy.server.handlers.util import hash_password -import uuid - - -STAGING_THREAD = concurrent.futures.ThreadPoolExecutor(max_workers=3) - - -class ContextLoggerWrapper: - """ - This class appends request context to logged messages. - """ - - @staticmethod - def _generate_call_id(): - return str(uuid.uuid4()) - - def __init__(self, request: tornado.httputil.HTTPServerRequest): - self.call_id = self._generate_call_id() - self.set_request(request) - - self.tabpy_username = None - self.log_request_context = False - self.request_context_logged = False - - def set_request(self, request: tornado.httputil.HTTPServerRequest): - """ - Set HTTP(S) request for logger. Headers will be used to - append request data as client information, Tableau user name, etc. - """ - self.remote_ip = request.remote_ip - self.method = request.method - self.url = request.full_url() - - self.client = request.headers.get("TabPy-Client", None) - self.tableau_username = request.headers.get("TabPy-User", None) - - def set_tabpy_username(self, tabpy_username: str): - self.tabpy_username = tabpy_username - - def enable_context_logging(self, enable: bool): - """ - Enable/disable request context information logging. - - Parameters - ---------- - enable: bool - If True request context information will be logged and - every log entry for a request handler will have call ID - with it. - """ - self.log_request_context = enable - - def _log_context_info(self): - if not self.log_request_context: - return - - context = f"Call ID: {self.call_id}" - - if self.remote_ip is not None: - context += f", Caller: {self.remote_ip}" - - if self.method is not None: - context += f", Method: {self.method}" - - if self.url is not None: - context += f", URL: {self.url}" - - if self.client is not None: - context += f", Client: {self.client}" - - if self.tableau_username is not None: - context += f", Tableau user: {self.tableau_username}" - - if self.tabpy_username is not None: - context += f", TabPy user: {self.tabpy_username}" - - logging.getLogger(__name__).log(logging.INFO, context) - self.request_context_logged = True - - def log(self, level: int, msg: str): - """ - Log message with or without call ID. If call context is logged and - call ID added to any log entry is specified by if context logging - is enabled (see CallContext.enable_context_logging for more details). - - Parameters - ---------- - level: int - Log level: logging.CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET. - - msg: str - Message format string. - - args - Same as args in Logger.debug(). - - kwargs - Same as kwargs in Logger.debug(). - """ - extended_msg = msg - if self.log_request_context: - if not self.request_context_logged: - self._log_context_info() - - extended_msg += f", <>" - - logging.getLogger(__name__).log(level, extended_msg) - - -class BaseHandler(tornado.web.RequestHandler): - def initialize(self, app): - self.tabpy_state = app.tabpy_state - # set content type to application/json - self.set_header("Content-Type", "application/json") - self.protocol = self.settings[SettingsParameters.TransferProtocol] - self.port = self.settings[SettingsParameters.Port] - self.python_service = app.python_service - self.credentials = app.credentials - self.username = None - self.password = None - self.eval_timeout = self.settings[SettingsParameters.EvaluateTimeout] - - self.logger = ContextLoggerWrapper(self.request) - self.logger.enable_context_logging( - app.settings[SettingsParameters.LogRequestContext] - ) - self.logger.log(logging.DEBUG, "Checking if need to handle authentication") - self.not_authorized = not self.handle_authentication("v1") - - def error_out(self, code, log_message, info=None): - self.set_status(code) - self.write(json.dumps({"message": log_message, "info": info or {}})) - - # We want to duplicate error message in console for - # loggers are misconfigured or causing the failure - # themselves - print(info) - self.logger.log( - logging.ERROR, - 'Responding with status={}, message="{}", info="{}"'.format( - code, log_message, info - ), - ) - self.finish() - - def options(self): - # add CORS headers if TabPy has a cors_origin specified - self._add_CORS_header() - self.write({}) - - def _add_CORS_header(self): - """ - Add CORS header if the TabPy has attribute _cors_origin - and _cors_origin is not an empty string. - """ - origin = self.tabpy_state.get_access_control_allow_origin() - if len(origin) > 0: - self.set_header("Access-Control-Allow-Origin", origin) - self.logger.log(logging.DEBUG, f"Access-Control-Allow-Origin:{origin}") - - headers = self.tabpy_state.get_access_control_allow_headers() - if len(headers) > 0: - self.set_header("Access-Control-Allow-Headers", headers) - self.logger.log(logging.DEBUG, f"Access-Control-Allow-Headers:{headers}") - - methods = self.tabpy_state.get_access_control_allow_methods() - if len(methods) > 0: - self.set_header("Access-Control-Allow-Methods", methods) - self.logger.log(logging.DEBUG, f"Access-Control-Allow-Methods:{methods}") - - def _get_auth_method(self, api_version) -> (bool, str): - """ - Finds authentication method if provided. - - Parameters - ---------- - api_version : str - API version for authentication. - - Returns - ------- - bool - True if known authentication method is found. - False otherwise. - - str - Name of authentication method used by client. - If empty no authentication required. - - (True, '') as result of this function means authentication - is not needed. - """ - if api_version not in self.settings[SettingsParameters.ApiVersions]: - self.logger.log(logging.CRITICAL, f'Unknown API version "{api_version}"') - return False, "" - - version_settings = self.settings[SettingsParameters.ApiVersions][api_version] - if "features" not in version_settings: - self.logger.log( - logging.INFO, f'No features configured for API "{api_version}"' - ) - return True, "" - - features = version_settings["features"] - if ( - "authentication" not in features - or not features["authentication"]["required"] - ): - self.logger.log( - logging.INFO, - "Authentication is not a required feature for API " f'"{api_version}"', - ) - return True, "" - - auth_feature = features["authentication"] - if "methods" not in auth_feature: - self.logger.log( - logging.INFO, - "Authentication method is not configured for API " f'"{api_version}"', - ) - - methods = auth_feature["methods"] - if "basic-auth" in auth_feature["methods"]: - return True, "basic-auth" - # Add new methods here... - - # No known methods were found - self.logger.log( - logging.CRITICAL, - f'Unknown authentication method(s) "{methods}" are configured ' - f'for API "{api_version}"', - ) - return False, "" - - def _get_basic_auth_credentials(self) -> bool: - """ - Find credentials for basic access authentication method. Credentials if - found stored in Credentials.username and Credentials.password. - - Returns - ------- - bool - True if valid credentials were found. - False otherwise. - """ - self.logger.log( - logging.DEBUG, "Checking request headers for authentication data" - ) - if "Authorization" not in self.request.headers: - self.logger.log(logging.INFO, "Authorization header not found") - return False - - auth_header = self.request.headers["Authorization"] - auth_header_list = auth_header.split(" ") - if len(auth_header_list) != 2 or auth_header_list[0] != "Basic": - self.logger.log( - logging.ERROR, f'Unknown authentication method "{auth_header}"' - ) - return False - - try: - cred = base64.b64decode(auth_header_list[1]).decode("utf-8") - except (binascii.Error, UnicodeDecodeError) as ex: - self.logger.log(logging.CRITICAL, f"Cannot decode credentials: {str(ex)}") - return False - - login_pwd = cred.split(":") - if len(login_pwd) != 2: - self.logger.log(logging.ERROR, "Invalid string in encoded credentials") - return False - - self.username = login_pwd[0] - self.logger.set_tabpy_username(self.username) - self.password = login_pwd[1] - return True - - def _get_credentials(self, method) -> bool: - """ - Find credentials for specified authentication method. Credentials if - found stored in self.username and self.password. - - Parameters - ---------- - method: str - Authentication method name. - - Returns - ------- - bool - True if valid credentials were found. - False otherwise. - """ - if method == "basic-auth": - return self._get_basic_auth_credentials() - # Add new methods here... - - # No known methods were found - self.logger.log( - logging.CRITICAL, - f'Unknown authentication method(s) "{method}" are configured ', - ) - return False - - def _validate_basic_auth_credentials(self) -> bool: - """ - Validates username:pwd if they are the same as - stored credentials. - - Returns - ------- - bool - True if credentials has key login and - credentials[login] equal SHA3(pwd), False - otherwise. - """ - login = self.username.lower() - self.logger.log( - logging.DEBUG, f'Validating credentials for user name "{login}"' - ) - if login not in self.credentials: - self.logger.log(logging.ERROR, f'User name "{self.username}" not found') - return False - - hashed_pwd = hash_password(login, self.password) - if self.credentials[login].lower() != hashed_pwd.lower(): - self.logger.log( - logging.ERROR, f'Wrong password for user name "{self.username}"' - ) - return False - - return True - - def _validate_credentials(self, method) -> bool: - """ - Validates credentials according to specified methods if they - are what expected. - - Parameters - ---------- - method: str - Authentication method name. - - Returns - ------- - bool - True if credentials are valid. - False otherwise. - """ - if method == "basic-auth": - return self._validate_basic_auth_credentials() - # Add new methods here... - - # No known methods were found - self.logger.log( - logging.CRITICAL, - f'Unknown authentication method(s) "{method}" are configured ', - ) - return False - - def handle_authentication(self, api_version) -> bool: - """ - If authentication feature is configured checks provided - credentials. - - Parameters - ---------- - api_version : str - API version for authentication. - - Returns - ------- - bool - True if authentication is not required. - True if authentication is required and valid - credentials provided. - False otherwise. - """ - self.logger.log(logging.DEBUG, "Handling authentication") - found, method = self._get_auth_method(api_version) - if not found: - return False - - if method == "": - # Do not validate credentials - return True - - if not self._get_credentials(method): - return False - - return self._validate_credentials(method) - - def should_fail_with_not_authorized(self): - """ - Checks if authentication is required: - - if it is not returns false, None - - if it is required validates provided credentials - - Returns - ------- - bool - False if authentication is not required or is - required and validation for credentials passes. - True if validation for credentials failed. - """ - return self.not_authorized - - def fail_with_not_authorized(self): - """ - Prepares server 401 response. - """ - self.logger.log(logging.ERROR, "Failing with 401 for unauthorized request") - self.set_status(401) - self.set_header("WWW-Authenticate", f'Basic realm="{self.tabpy_state.name}"') - self.error_out( - 401, - info="Unauthorized request.", - log_message="Invalid credentials provided.", - ) diff --git a/tabpy/server/handlers/endpoint_handler.py b/tabpy/server/handlers/endpoint_handler.py deleted file mode 100644 index 2f882ef8..00000000 --- a/tabpy/server/handlers/endpoint_handler.py +++ /dev/null @@ -1,141 +0,0 @@ -""" -HTTP handeler to serve specific endpoint request like -http://myserver:9004/endpoints/mymodel - -For how generic endpoints requests is served look -at endpoints_handler.py -""" - -import json -import logging -import shutil -from tabpy.server.common.util import format_exception -from tabpy.server.handlers import ManagementHandler -from tabpy.server.handlers.base_handler import STAGING_THREAD -from tabpy.server.management.state import get_query_object_path -from tabpy.server.psws.callbacks import on_state_change -from tornado import gen - - -class EndpointHandler(ManagementHandler): - def initialize(self, app): - super(EndpointHandler, self).initialize(app) - - def get(self, endpoint_name): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - self.logger.log(logging.DEBUG, f"Processing GET for /endpoints/{endpoint_name}") - - self._add_CORS_header() - if not endpoint_name: - self.write(json.dumps(self.tabpy_state.get_endpoints())) - else: - if endpoint_name in self.tabpy_state.get_endpoints(): - self.write(json.dumps(self.tabpy_state.get_endpoints()[endpoint_name])) - else: - self.error_out( - 404, - "Unknown endpoint", - info=f"Endpoint {endpoint_name} is not found", - ) - - @gen.coroutine - def put(self, name): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - self.logger.log(logging.DEBUG, f"Processing PUT for /endpoints/{name}") - - try: - if not self.request.body: - self.error_out(400, "Input body cannot be empty") - self.finish() - return - try: - request_data = json.loads(self.request.body.decode("utf-8")) - except BaseException as ex: - self.error_out( - 400, log_message="Failed to decode input body", info=str(ex) - ) - self.finish() - return - - # check if endpoint exists - endpoints = self.tabpy_state.get_endpoints(name) - if len(endpoints) == 0: - self.error_out(404, f"endpoint {name} does not exist.") - self.finish() - return - - new_version = int(endpoints[name]["version"]) + 1 - self.logger.log(logging.INFO, f"Endpoint info: {request_data}") - err_msg = yield self._add_or_update_endpoint( - "update", name, new_version, request_data - ) - if err_msg: - self.error_out(400, err_msg) - self.finish() - else: - self.write(self.tabpy_state.get_endpoints(name)) - self.finish() - - except Exception as e: - err_msg = format_exception(e, "update_endpoint") - self.error_out(500, err_msg) - self.finish() - - @gen.coroutine - def delete(self, name): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - self.logger.log(logging.DEBUG, f"Processing DELETE for /endpoints/{name}") - - try: - endpoints = self.tabpy_state.get_endpoints(name) - if len(endpoints) == 0: - self.error_out(404, f"endpoint {name} does not exist.") - self.finish() - return - - # update state - try: - endpoint_info = self.tabpy_state.delete_endpoint(name) - except Exception as e: - self.error_out(400, f"Error when removing endpoint: {e.message}") - self.finish() - return - - # delete files - if endpoint_info["type"] != "alias": - delete_path = get_query_object_path( - self.settings["state_file_path"], name, None - ) - try: - yield self._delete_po_future(delete_path) - except Exception as e: - self.error_out(400, f"Error while deleting: {e}") - self.finish() - return - - self.set_status(204) - self.finish() - - except Exception as e: - err_msg = format_exception(e, "delete endpoint") - self.error_out(500, err_msg) - self.finish() - - on_state_change( - self.settings, self.tabpy_state, self.python_service, self.logger - ) - - @gen.coroutine - def _delete_po_future(self, delete_path): - future = STAGING_THREAD.submit(shutil.rmtree, delete_path) - ret = yield future - raise gen.Return(ret) diff --git a/tabpy/server/handlers/endpoints_handler.py b/tabpy/server/handlers/endpoints_handler.py deleted file mode 100644 index 0bd17c16..00000000 --- a/tabpy/server/handlers/endpoints_handler.py +++ /dev/null @@ -1,75 +0,0 @@ -""" -HTTP handeler to serve general endpoints request, specifically -http://myserver:9004/endpoints - -For how individual endpoint requests are served look -at endpoint_handler.py -""" - -import json -import logging -from tabpy.server.common.util import format_exception -from tabpy.server.handlers import ManagementHandler -from tornado import gen - - -class EndpointsHandler(ManagementHandler): - def initialize(self, app): - super(EndpointsHandler, self).initialize(app) - - def get(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - self._add_CORS_header() - self.write(json.dumps(self.tabpy_state.get_endpoints())) - - @gen.coroutine - def post(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - try: - if not self.request.body: - self.error_out(400, "Input body cannot be empty") - self.finish() - return - - try: - request_data = json.loads(self.request.body.decode("utf-8")) - except Exception as ex: - self.error_out(400, "Failed to decode input body", str(ex)) - self.finish() - return - - if "name" not in request_data: - self.error_out(400, "name is required to add an endpoint.") - self.finish() - return - - name = request_data["name"] - - # check if endpoint already exist - if name in self.tabpy_state.get_endpoints(): - self.error_out(400, f"endpoint {name} already exists.") - self.finish() - return - - self.logger.log(logging.DEBUG, f'Adding endpoint "{name}"') - err_msg = yield self._add_or_update_endpoint("add", name, 1, request_data) - if err_msg: - self.error_out(400, err_msg) - else: - self.logger.log(logging.DEBUG, f"Endpoint {name} successfully added") - self.set_status(201) - self.write(self.tabpy_state.get_endpoints(name)) - self.finish() - return - - except Exception as e: - err_msg = format_exception(e, "/add_endpoint") - self.error_out(500, "error adding endpoint", err_msg) - self.finish() - return diff --git a/tabpy/server/handlers/evaluation_plane_handler.py b/tabpy/server/handlers/evaluation_plane_handler.py deleted file mode 100644 index 3e73e359..00000000 --- a/tabpy/server/handlers/evaluation_plane_handler.py +++ /dev/null @@ -1,142 +0,0 @@ -from tabpy.server.handlers import BaseHandler -import json -import simplejson -import logging -from tabpy.server.common.util import format_exception -import requests -from tornado import gen -from datetime import timedelta - - -class RestrictedTabPy: - def __init__(self, protocol, port, logger, timeout): - self.protocol = protocol - self.port = port - self.logger = logger - self.timeout = timeout - - def query(self, name, *args, **kwargs): - url = f"{self.protocol}://localhost:{self.port}/query/{name}" - self.logger.log(logging.DEBUG, f"Querying {url}...") - internal_data = {"data": args or kwargs} - data = json.dumps(internal_data) - headers = {"content-type": "application/json"} - response = requests.post( - url=url, data=data, headers=headers, timeout=self.timeout, verify=False - ) - return response.json() - - -class EvaluationPlaneHandler(BaseHandler): - """ - EvaluationPlaneHandler is responsible for running arbitrary python scripts. - """ - - def initialize(self, executor, app): - super(EvaluationPlaneHandler, self).initialize(app) - self.executor = executor - self._error_message_timeout = ( - f"User defined script timed out. " - f"Timeout is set to {self.eval_timeout} s." - ) - - @gen.coroutine - def _post_impl(self): - body = json.loads(self.request.body.decode("utf-8")) - self.logger.log(logging.DEBUG, f"Processing POST request '{body}'...") - if "script" not in body: - self.error_out(400, "Script is empty.") - return - - # Transforming user script into a proper function. - user_code = body["script"] - arguments = None - arguments_str = "" - if "data" in body: - arguments = body["data"] - - if arguments is not None: - if not isinstance(arguments, dict): - self.error_out( - 400, "Script parameters need to be provided as a dictionary." - ) - return - args_in = sorted(arguments.keys()) - n = len(arguments) - if sorted('_arg'+str(i+1) for i in range(n)) == args_in: - arguments_str = ", " + ", ".join(args_in) - else: - self.error_out( - 400, - "Variables names should follow " - "the format _arg1, _arg2, _argN", - ) - return - - function_to_evaluate = f"def _user_script(tabpy{arguments_str}):\n" - for u in user_code.splitlines(): - function_to_evaluate += " " + u + "\n" - - self.logger.log( - logging.INFO, f"function to evaluate={function_to_evaluate}" - ) - - try: - result = yield self._call_subprocess(function_to_evaluate, arguments) - except ( - gen.TimeoutError, - requests.exceptions.ConnectTimeout, - requests.exceptions.ReadTimeout, - ): - self.logger.log(logging.ERROR, self._error_message_timeout) - self.error_out(408, self._error_message_timeout) - return - - if result is not None: - self.write(simplejson.dumps(result, ignore_nan=True)) - else: - self.write("null") - self.finish() - - @gen.coroutine - def post(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - self._add_CORS_header() - try: - yield self._post_impl() - except Exception as e: - err_msg = f"{e.__class__.__name__} : {str(e)}" - if err_msg != "KeyError : 'response'": - err_msg = format_exception(e, "POST /evaluate") - self.error_out(500, "Error processing script", info=err_msg) - else: - self.error_out( - 404, - "Error processing script", - info="The endpoint you're " - "trying to query did not respond. Please make sure the " - "endpoint exists and the correct set of arguments are " - "provided.", - ) - - @gen.coroutine - def _call_subprocess(self, function_to_evaluate, arguments): - restricted_tabpy = RestrictedTabPy( - self.protocol, self.port, self.logger, self.eval_timeout - ) - # Exec does not run the function, so it does not block. - exec(function_to_evaluate, globals()) - - # 'noqa' comments below tell flake8 to ignore undefined _user_script - # name - the name is actually defined with user script being wrapped - # in _user_script function (constructed as a striong) and then executed - # with exec() call above. - future = self.executor.submit(_user_script, # noqa: F821 - restricted_tabpy, - **arguments if arguments is not None else None) - - ret = yield gen.with_timeout(timedelta(seconds=self.eval_timeout), future) - raise gen.Return(ret) diff --git a/tabpy/server/handlers/main_handler.py b/tabpy/server/handlers/main_handler.py deleted file mode 100644 index d6a43025..00000000 --- a/tabpy/server/handlers/main_handler.py +++ /dev/null @@ -1,7 +0,0 @@ -from tabpy.server.handlers import BaseHandler - - -class MainHandler(BaseHandler): - def get(self): - self._add_CORS_header() - self.render("/static/index.html") diff --git a/tabpy/server/handlers/management_handler.py b/tabpy/server/handlers/management_handler.py deleted file mode 100644 index b24812dd..00000000 --- a/tabpy/server/handlers/management_handler.py +++ /dev/null @@ -1,150 +0,0 @@ -import logging -import os -import shutil -from re import compile as _compile -from uuid import uuid4 as random_uuid - -from tornado import gen - -from tabpy.server.app.SettingsParameters import SettingsParameters -from tabpy.server.handlers import MainHandler -from tabpy.server.handlers.base_handler import STAGING_THREAD -from tabpy.server.management.state import get_query_object_path -from tabpy.server.psws.callbacks import on_state_change - - -def copy_from_local(localpath, remotepath, is_dir=False): - if is_dir: - if not os.path.exists(remotepath): - # remote folder does not exist - shutil.copytree(localpath, remotepath) - else: - # remote folder exists, copy each file - src_files = os.listdir(localpath) - for file_name in src_files: - full_file_name = os.path.join(localpath, file_name) - if os.path.isdir(full_file_name): - # copy folder recursively - full_remote_path = os.path.join(remotepath, file_name) - shutil.copytree(full_file_name, full_remote_path) - else: - # copy each file - shutil.copy(full_file_name, remotepath) - else: - shutil.copy(localpath, remotepath) - - -class ManagementHandler(MainHandler): - def initialize(self, app): - super(ManagementHandler, self).initialize(app) - self.port = self.settings[SettingsParameters.Port] - - def _get_protocol(self): - return "http://" - - @gen.coroutine - def _add_or_update_endpoint(self, action, name, version, request_data): - """ - Add or update an endpoint - """ - self.logger.log(logging.DEBUG, f"Adding/updating model {name}...") - - if not isinstance(name, str): - msg = "Endpoint name must be a string" - self.logger.log(logging.CRITICAL, msg) - raise TypeError(msg) - - name_checker = _compile(r"^[a-zA-Z0-9-_\s]+$") - if not name_checker.match(name): - raise gen.Return( - "endpoint name can only contain: a-z, A-Z, 0-9," - " underscore, hyphens and spaces." - ) - - if self.settings.get("add_or_updating_endpoint"): - msg = ( - "Another endpoint update is already in progress" - ", please wait a while and try again" - ) - self.logger.log(logging.CRITICAL, msg) - raise RuntimeError(msg) - - self.settings["add_or_updating_endpoint"] = random_uuid() - try: - docstring = None - if "docstring" in request_data: - docstring = str( - bytes(request_data["docstring"], "utf-8").decode("unicode_escape") - ) - - description = request_data.get("description", None) - endpoint_type = request_data.get("type", None) - methods = request_data.get("methods", []) - dependencies = request_data.get("dependencies", None) - target = request_data.get("target", None) - schema = request_data.get("schema", None) - src_path = request_data.get("src_path", None) - target_path = get_query_object_path( - self.settings[SettingsParameters.StateFilePath], name, version - ) - - path_checker = _compile(r"^[\\\:a-zA-Z0-9-_~\s/\.\(\)]+$") - # copy from staging - if src_path: - if not isinstance(src_path, str): - raise gen.Return("src_path must be a string.") - if not path_checker.match(src_path): - raise gen.Return(f"Invalid source path for endpoint {name}") - - yield self._copy_po_future(src_path, target_path) - elif endpoint_type != "alias": - raise gen.Return("src_path is required to add/update an endpoint.") - else: - # alias special logic: - if not target: - raise gen.Return("Target is required for alias endpoint.") - dependencies = [target] - - # update local config - try: - if action == "add": - self.tabpy_state.add_endpoint( - name=name, - description=description, - docstring=docstring, - endpoint_type=endpoint_type, - methods=methods, - dependencies=dependencies, - target=target, - schema=schema, - ) - else: - self.tabpy_state.update_endpoint( - name=name, - description=description, - docstring=docstring, - endpoint_type=endpoint_type, - methods=methods, - dependencies=dependencies, - target=target, - schema=schema, - version=version, - ) - - except Exception as e: - raise gen.Return(f"Error when changing TabPy state: {e}") - - on_state_change( - self.settings, self.tabpy_state, self.python_service, self.logger - ) - - finally: - self.settings["add_or_updating_endpoint"] = None - - @gen.coroutine - def _copy_po_future(self, src_path, target_path): - future = STAGING_THREAD.submit( - copy_from_local, src_path, target_path, is_dir=True - ) - ret = yield future - raise gen.Return(ret) diff --git a/tabpy/server/handlers/query_plane_handler.py b/tabpy/server/handlers/query_plane_handler.py deleted file mode 100644 index 16928b4f..00000000 --- a/tabpy/server/handlers/query_plane_handler.py +++ /dev/null @@ -1,233 +0,0 @@ -from tabpy.server.handlers import BaseHandler -import logging -import time -from tabpy.server.common.messages import ( - Query, - QuerySuccessful, - QueryError, - UnknownURI, -) -from hashlib import md5 -import uuid -import json -from tabpy.server.common.util import format_exception -import urllib -from tornado import gen - - -def _get_uuid(): - """Generate a unique identifier string""" - return str(uuid.uuid4()) - - -class QueryPlaneHandler(BaseHandler): - def initialize(self, app): - super(QueryPlaneHandler, self).initialize(app) - - def _query(self, po_name, data, uid, qry): - """ - Parameters - ---------- - po_name : str - The name of the query object to query - - data : dict - The deserialized request body - - uid: str - A unique identifier for the request - - qry: str - The incoming query object. This object maintains - raw incoming request, which is different from the sanitied data - - Returns - ------- - out : (result type, dict, int) - A triple containing a result type, the result message - as a dictionary, and the time in seconds that it took to complete - the request. - """ - self.logger.log(logging.DEBUG, f"Collecting query info for {po_name}...") - start_time = time.time() - response = self.python_service.ps.query(po_name, data, uid) - gls_time = time.time() - start_time - self.logger.log(logging.DEBUG, f"Query info: {response}") - - if isinstance(response, QuerySuccessful): - response_json = response.to_json() - md5_tag = md5(response_json.encode("utf-8")).hexdigest() - self.set_header("Etag", f'"{md5_tag}"') - return (QuerySuccessful, response.for_json(), gls_time) - else: - self.logger.log(logging.ERROR, f"Failed query, response: {response}") - return (type(response), response.for_json(), gls_time) - - # handle HTTP Options requests to support CORS - # don't check API key (client does not send or receive data for OPTIONS, - # it just allows the client to subsequently make a POST request) - def options(self, pred_name): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - self.logger.log(logging.DEBUG, f"Processing OPTIONS for /query/{pred_name}") - - # add CORS headers if TabPy has a cors_origin specified - self._add_CORS_header() - self.write({}) - - def _handle_result(self, po_name, data, qry, uid): - (response_type, response, gls_time) = self._query(po_name, data, uid, qry) - - if response_type == QuerySuccessful: - result_dict = { - "response": response["response"], - "version": response["version"], - "model": po_name, - "uuid": uid, - } - self.write(result_dict) - self.finish() - return (gls_time, response["response"]) - else: - if response_type == UnknownURI: - self.error_out( - 404, - "UnknownURI", - info=( - "No query object has been registered" - f' with the name "{po_name}"' - ), - ) - elif response_type == QueryError: - self.error_out(400, "QueryError", info=response) - else: - self.error_out(500, "Error querying GLS", info=response) - - return (None, None) - - def _sanitize_request_data(self, data): - if not isinstance(data, dict): - msg = "Input data must be a dictionary" - self.logger.log(logging.CRITICAL, msg) - raise RuntimeError(msg) - - if "method" in data: - return {"data": data.get("data"), "method": data.get("method")} - elif "data" in data: - return data.get("data") - else: - msg = 'Input data must be a dictionary with a key called "data"' - self.logger.log(logging.CRITICAL, msg) - raise RuntimeError(msg) - - def _process_query(self, endpoint_name, start): - self.logger.log(logging.DEBUG, f"Processing query {endpoint_name}...") - try: - self._add_CORS_header() - - if not self.request.body: - self.request.body = {} - - # extract request data explicitly for caching purpose - request_json = self.request.body.decode("utf-8") - - # Sanitize input data - data = self._sanitize_request_data(json.loads(request_json)) - except Exception as e: - self.logger.log(logging.ERROR, str(e)) - err_msg = format_exception(e, "Invalid Input Data") - self.error_out(400, err_msg) - return - - try: - (po_name, _) = self._get_actual_model(endpoint_name) - - # po_name is None if self.python_service.ps.query_objects.get( - # endpoint_name) is None - if not po_name: - self.error_out( - 404, "UnknownURI", info=f'Endpoint "{endpoint_name}" does not exist' - ) - return - - po_obj = self.python_service.ps.query_objects.get(po_name) - - if not po_obj: - self.error_out( - 404, "UnknownURI", info=f'Endpoint "{po_name}" does not exist' - ) - return - - if po_name != endpoint_name: - self.logger.log( - logging.INFO, f"Querying actual model: po_name={po_name}" - ) - - uid = _get_uuid() - - # record query w/ request ID in query log - qry = Query(po_name, request_json) - gls_time = 0 - # send a query to PythonService and return - (gls_time, _) = self._handle_result(po_name, data, qry, uid) - - # if error occurred, GLS time is None. - if not gls_time: - return - - except Exception as e: - self.logger.log(logging.ERROR, str(e)) - err_msg = format_exception(e, "process query") - self.error_out(500, "Error processing query", info=err_msg) - return - - def _get_actual_model(self, endpoint_name): - # Find the actual query to run from given endpoint - all_endpoint_names = [] - - while True: - endpoint_info = self.python_service.ps.query_objects.get(endpoint_name) - if not endpoint_info: - return [None, None] - - all_endpoint_names.append(endpoint_name) - - endpoint_type = endpoint_info.get("type", "model") - - if endpoint_type == "alias": - endpoint_name = endpoint_info["endpoint_obj"] - elif endpoint_type == "model": - break - else: - self.error_out( - 500, - "Unknown endpoint type", - info=f'Endpoint type "{endpoint_type}" does not exist', - ) - return - - return (endpoint_name, all_endpoint_names) - - @gen.coroutine - def get(self, endpoint_name): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - start = time.time() - endpoint_name = urllib.parse.unquote(endpoint_name) - self._process_query(endpoint_name, start) - - @gen.coroutine - def post(self, endpoint_name): - self.logger.log(logging.DEBUG, f"Processing POST for /query/{endpoint_name}...") - - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - start = time.time() - endpoint_name = urllib.parse.unquote(endpoint_name) - self._process_query(endpoint_name, start) diff --git a/tabpy/server/handlers/service_info_handler.py b/tabpy/server/handlers/service_info_handler.py deleted file mode 100644 index 2451d199..00000000 --- a/tabpy/server/handlers/service_info_handler.py +++ /dev/null @@ -1,23 +0,0 @@ -import json -from tabpy.server.app.SettingsParameters import SettingsParameters -from tabpy.server.handlers import ManagementHandler - - -class ServiceInfoHandler(ManagementHandler): - def initialize(self, app): - super(ServiceInfoHandler, self).initialize(app) - - def get(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - self._add_CORS_header() - info = {} - info["description"] = self.tabpy_state.get_description() - info["creation_time"] = self.tabpy_state.creation_time - info["state_path"] = self.settings[SettingsParameters.StateFilePath] - info["server_version"] = self.settings[SettingsParameters.ServerVersion] - info["name"] = self.tabpy_state.name - info["versions"] = self.settings[SettingsParameters.ApiVersions] - self.write(json.dumps(info)) diff --git a/tabpy/server/handlers/status_handler.py b/tabpy/server/handlers/status_handler.py deleted file mode 100644 index 622e0199..00000000 --- a/tabpy/server/handlers/status_handler.py +++ /dev/null @@ -1,29 +0,0 @@ -import json -import logging -from tabpy.server.handlers import BaseHandler - - -class StatusHandler(BaseHandler): - def initialize(self, app): - super(StatusHandler, self).initialize(app) - - def get(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - self._add_CORS_header() - - status_dict = {} - for k, v in self.python_service.ps.query_objects.items(): - status_dict[k] = { - "version": v["version"], - "type": v["type"], - "status": v["status"], - "last_error": v["last_error"], - } - - self.logger.log(logging.DEBUG, f"Found models: {status_dict}") - self.write(json.dumps(status_dict)) - self.finish() - return diff --git a/tabpy/server/handlers/upload_destination_handler.py b/tabpy/server/handlers/upload_destination_handler.py deleted file mode 100644 index a04dad8e..00000000 --- a/tabpy/server/handlers/upload_destination_handler.py +++ /dev/null @@ -1,20 +0,0 @@ -from tabpy.server.app.SettingsParameters import SettingsParameters -from tabpy.server.handlers import ManagementHandler -import os - - -_QUERY_OBJECT_STAGING_FOLDER = "staging" - - -class UploadDestinationHandler(ManagementHandler): - def initialize(self, app): - super(UploadDestinationHandler, self).initialize(app) - - def get(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() - return - - path = self.settings[SettingsParameters.StateFilePath] - path = os.path.join(path, _QUERY_OBJECT_STAGING_FOLDER) - self.write({"path": path}) diff --git a/tabpy/server/handlers/util.py b/tabpy/server/handlers/util.py deleted file mode 100644 index 14e029c6..00000000 --- a/tabpy/server/handlers/util.py +++ /dev/null @@ -1,32 +0,0 @@ -import binascii -from hashlib import pbkdf2_hmac - - -def hash_password(username, pwd): - """ - Hashes password using PKDBF2 method: - hash = PKDBF2('sha512', pwd, salt=username, 10000) - - Parameters - ---------- - username : str - User name (login). Used as salt for hashing. - User name is lowercased befor being used in hashing. - Salt is formatted as '_$salt@tabpy:$_' to - guarantee there's at least 16 characters. - - pwd : str - Password to hash. - - Returns - ------- - str - Sting representation (hexidecimal) for PBKDF2 hash - for the password. - """ - salt = f"_$salt@tabpy:{username.lower()}$_" - - hash = pbkdf2_hmac( - hash_name="sha512", password=pwd.encode(), salt=salt.encode(), iterations=10000 - ) - return binascii.hexlify(hash).decode() diff --git a/tabpy/server/management/__init__.py b/tabpy/server/management/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tabpy/server/management/state.py b/tabpy/server/management/state.py deleted file mode 100644 index 697d24fd..00000000 --- a/tabpy/server/management/state.py +++ /dev/null @@ -1,643 +0,0 @@ -try: - from ConfigParser import ConfigParser -except ImportError: - from configparser import ConfigParser -import json -import logging -from tabpy.server.management.util import write_state_config -from threading import Lock -from time import time - - -logger = logging.getLogger(__name__) - -# State File Config Section Names -_DEPLOYMENT_SECTION_NAME = "Query Objects Service Versions" -_QUERY_OBJECT_DOCSTRING = "Query Objects Docstrings" -_SERVICE_INFO_SECTION_NAME = "Service Info" -_META_SECTION_NAME = "Meta" - -# Directory Names -_QUERY_OBJECT_DIR = "query_objects" - -""" -Lock to change the TabPy State. -""" -_PS_STATE_LOCK = Lock() - - -def state_lock(func): - """ - Mutex for changing PS state - """ - - def wrapper(self, *args, **kwargs): - try: - _PS_STATE_LOCK.acquire() - return func(self, *args, **kwargs) - finally: - # ALWAYS RELEASE LOCK - _PS_STATE_LOCK.release() - - return wrapper - - -def _get_root_path(state_path): - if state_path[-1] != "/": - state_path += "/" - - return state_path - - -def get_query_object_path(state_file_path, name, version): - """ - Returns the query object path - - If the version is None, a path without the version will be returned. - """ - root_path = _get_root_path(state_file_path) - sub_path = [_QUERY_OBJECT_DIR, name] - if version is not None: - sub_path.append(str(version)) - full_path = root_path + "/".join(sub_path) - return full_path - - -class TabPyState: - """ - The TabPy state object that stores attributes - about this TabPy and perform GET/SET on these - attributes. - - Attributes: - - name - - description - - endpoints (name, description, docstring, version, target) - - revision number - - When the state object is initialized, the state is saved as a ConfigParser. - There is a config to any attribute. - - """ - - def __init__(self, settings, config=None): - self.settings = settings - self.set_config(config, _update=False) - - @state_lock - def set_config(self, config, logger=logging.getLogger(__name__), _update=True): - """ - Set the local ConfigParser manually. - This new ConfigParser will be used as current state. - """ - if not isinstance(config, ConfigParser): - raise ValueError("Invalid config") - self.config = config - if _update: - self._write_state(logger) - - def get_endpoints(self, name=None): - """ - Return a dictionary of endpoints - - Parameters - ---------- - name : str - The name of the endpoint. - If "name" is specified, only the information about that endpoint - will be returned. - - Returns - ------- - endpoints : dict - The dictionary containing information about each endpoint. - The keys are the endpoint names. - The values for each include: - - description - - doc string - - type - - target - - """ - endpoints = {} - try: - endpoint_names = self._get_config_value(_DEPLOYMENT_SECTION_NAME, name) - except Exception as e: - logger.error(f"error in get_endpoints: {str(e)}") - return {} - - if name: - endpoint_info = json.loads(endpoint_names) - docstring = self._get_config_value(_QUERY_OBJECT_DOCSTRING, name) - endpoint_info["docstring"] = str( - bytes(docstring, "utf-8").decode("unicode_escape") - ) - endpoints = {name: endpoint_info} - else: - for endpoint_name in endpoint_names: - endpoint_info = json.loads( - self._get_config_value(_DEPLOYMENT_SECTION_NAME, endpoint_name) - ) - docstring = self._get_config_value( - _QUERY_OBJECT_DOCSTRING, endpoint_name, True, "" - ) - endpoint_info["docstring"] = str( - bytes(docstring, "utf-8").decode("unicode_escape") - ) - endpoints[endpoint_name] = endpoint_info - logger.debug(f"Collected endpoints: {endpoints}") - return endpoints - - def _check_endpoint_exists(self, name): - endpoints = self.get_endpoints() - if not name or not isinstance(name, str) or len(name) == 0: - raise ValueError("name of the endpoint must be a valid string.") - - return name in endpoints - - def _check_and_set_endpoint_str_value(self, param, paramName, defaultValue): - if not param and defaultValue is not None: - return defaultValue - - if not param or not isinstance(param, str): - raise ValueError(f"{paramName} must be a string.") - - return param - - def _check_and_set_endpoint_description(self, description, defaultValue): - return self._check_and_set_endpoint_str_value(description, "description", defaultValue) - - def _check_and_set_endpoint_docstring(self, docstring, defaultValue): - return self._check_and_set_endpoint_str_value(docstring, "docstring", defaultValue) - - def _check_and_set_endpoint_type(self, endpoint_type, defaultValue): - return self._check_and_set_endpoint_str_value( - endpoint_type, "endpoint type", defaultValue) - - def _check_and_set_target(self, target, defaultValue): - return self._check_and_set_endpoint_str_value( - target, "target", defaultValue) - - def _check_and_set_dependencies(self, dependencies, defaultValue): - if not dependencies: - return defaultValue - - if dependencies or not isinstance(dependencies, list): - raise ValueError("dependencies must be a list.") - - return dependencies - - @state_lock - def add_endpoint( - self, - name, - description=None, - docstring=None, - endpoint_type=None, - methods=None, - target=None, - dependencies=None, - schema=None, - ): - """ - Add a new endpoint to the TabPy. - - Parameters - ---------- - name : str - Name of the endpoint - description : str, optional - Description of this endpoint - doc_string : str, optional - The doc string for this endpoint, if needed. - endpoint_type : str - The endpoint type (model, alias) - target : str, optional - The target endpoint name for the alias to be added. - - Note: - The version of this endpoint will be set to 1 since it is a new - endpoint. - - """ - try: - if (self._check_endpoint_exists(name)): - raise ValueError(f"endpoint {name} already exists.") - - endpoints = self.get_endpoints() - - description = self._check_and_set_endpoint_description(description, "") - docstring = self._check_and_set_endpoint_docstring( - docstring, "-- no docstring found in query function --") - endpoint_type = self._check_and_set_endpoint_type(endpoint_type, None) - dependencies = self._check_and_set_dependencies(dependencies, []) - - target = self._check_and_set_target(target, "") - if target and target not in endpoints: - raise ValueError("target endpoint is not valid.") - - endpoint_info = { - "description": description, - "docstring": docstring, - "type": endpoint_type, - "version": 1, - "dependencies": dependencies, - "target": target, - "creation_time": int(time()), - "last_modified_time": int(time()), - "schema": schema, - } - - endpoints[name] = endpoint_info - self._add_update_endpoints_config(endpoints) - except Exception as e: - logger.error(f"Error in add_endpoint: {e}") - raise - - def _add_update_endpoints_config(self, endpoints): - # save the endpoint info to config - dstring = "" - for endpoint_name in endpoints: - try: - info = endpoints[endpoint_name] - dstring = str( - bytes(info["docstring"], "utf-8").decode("unicode_escape") - ) - self._set_config_value( - _QUERY_OBJECT_DOCSTRING, - endpoint_name, - dstring, - _update_revision=False, - ) - del info["docstring"] - self._set_config_value( - _DEPLOYMENT_SECTION_NAME, endpoint_name, json.dumps(info) - ) - except Exception as e: - logger.error(f"Unable to write endpoints config: {e}") - raise - - @state_lock - def update_endpoint( - self, - name, - description=None, - docstring=None, - endpoint_type=None, - version=None, - methods=None, - target=None, - dependencies=None, - schema=None, - ): - """ - Update an existing endpoint on the TabPy. - - Parameters - ---------- - name : str - Name of the endpoint - description : str, optional - Description of this endpoint - doc_string : str, optional - The doc string for this endpoint, if needed. - endpoint_type : str, optional - The endpoint type (model, alias) - version : str, optional - The version of this endpoint - dependencies=[] - List of dependent endpoints for this existing endpoint - target : str, optional - The target endpoint name for the alias. - - Note: - For those parameters that are not specified, those values will not - get changed. - - """ - try: - if (not self._check_endpoint_exists(name)): - raise ValueError(f"endpoint {name} does not exist.") - - endpoints = self.get_endpoints() - endpoint_info = endpoints[name] - - description = self._check_and_set_endpoint_description( - description, endpoint_info["description"]) - docstring = self._check_and_set_endpoint_docstring( - docstring, endpoint_info["docstring"]) - endpoint_type = self._check_and_set_endpoint_type( - endpoint_type, endpoint_info["type"]) - dependencies = self._check_and_set_dependencies( - dependencies, endpoint_info.get("dependencies", [])) - - target = self._check_and_set_target(target, None) - if target and target not in endpoints: - raise ValueError("target endpoint is not valid.") - elif not target: - target = endpoint_info["target"] - - if version and not isinstance(version, int): - raise ValueError("version must be an int.") - elif not version: - version = endpoint_info["version"] - - endpoint_info = { - "description": description, - "docstring": docstring, - "type": endpoint_type, - "version": version, - "dependencies": dependencies, - "target": target, - "creation_time": endpoint_info["creation_time"], - "last_modified_time": int(time()), - "schema": schema, - } - - endpoints[name] = endpoint_info - self._add_update_endpoints_config(endpoints) - except Exception as e: - logger.error(f"Error in update_endpoint: {e}") - raise - - @state_lock - def delete_endpoint(self, name): - """ - Delete an existing endpoint on the TabPy - - Parameters - ---------- - name : str - The name of the endpoint to be deleted. - - Returns - ------- - deleted endpoint object - - Note: - Cannot delete this endpoint if other endpoints are currently - depending on this endpoint. - - """ - if not name or name == "": - raise ValueError("Name of the endpoint must be a valid string.") - endpoints = self.get_endpoints() - if name not in endpoints: - raise ValueError(f"Endpoint {name} does not exist.") - - endpoint_to_delete = endpoints[name] - - # get dependencies and target - deps = set() - for endpoint_name in endpoints: - if endpoint_name != name: - deps_list = endpoints[endpoint_name].get("dependencies", []) - if name in deps_list: - deps.add(endpoint_name) - - # check if other endpoints are depending on this endpoint - if len(deps) > 0: - raise ValueError( - f"Cannot remove endpoint {name}, it is currently " - f"used by {list(deps)} endpoints." - ) - - del endpoints[name] - - # delete the endpoint from state - try: - self._remove_config_option( - _QUERY_OBJECT_DOCSTRING, name, _update_revision=False - ) - self._remove_config_option(_DEPLOYMENT_SECTION_NAME, name) - - return endpoint_to_delete - except Exception as e: - logger.error(f"Unable to delete endpoint {e}") - raise ValueError(f"Unable to delete endpoint: {e}") - - @property - def name(self): - """ - Returns the name of the TabPy service. - """ - name = None - try: - name = self._get_config_value(_SERVICE_INFO_SECTION_NAME, "Name") - except Exception as e: - logger.error(f"Unable to get name: {e}") - return name - - @property - def creation_time(self): - """ - Returns the creation time of the TabPy service. - """ - creation_time = 0 - try: - creation_time = self._get_config_value( - _SERVICE_INFO_SECTION_NAME, "Creation Time" - ) - except Exception as e: - logger.error(f"Unable to get name: {e}") - return creation_time - - @state_lock - def set_name(self, name): - """ - Set the name of this TabPy service. - - Parameters - ---------- - name : str - Name of TabPy service. - """ - if not isinstance(name, str): - raise ValueError("name must be a string.") - try: - self._set_config_value(_SERVICE_INFO_SECTION_NAME, "Name", name) - except Exception as e: - logger.error(f"Unable to set name: {e}") - - def get_description(self): - """ - Returns the description of the TabPy service. - """ - description = None - try: - description = self._get_config_value( - _SERVICE_INFO_SECTION_NAME, "Description" - ) - except Exception as e: - logger.error(f"Unable to get description: {e}") - return description - - @state_lock - def set_description(self, description): - """ - Set the description of this TabPy service. - - Parameters - ---------- - description : str - Description of TabPy service. - """ - if not isinstance(description, str): - raise ValueError("Description must be a string.") - try: - self._set_config_value( - _SERVICE_INFO_SECTION_NAME, "Description", description - ) - except Exception as e: - logger.error(f"Unable to set description: {e}") - - def get_revision_number(self): - """ - Returns the revision number of this TabPy service. - """ - rev = -1 - try: - rev = int(self._get_config_value(_META_SECTION_NAME, "Revision Number")) - except Exception as e: - logger.error(f"Unable to get revision number: {e}") - return rev - - def get_access_control_allow_origin(self): - """ - Returns Access-Control-Allow-Origin of this TabPy service. - """ - _cors_origin = "" - try: - logger.debug("Collecting Access-Control-Allow-Origin from state file ...") - _cors_origin = self._get_config_value( - "Service Info", "Access-Control-Allow-Origin" - ) - except Exception as e: - logger.error(e) - return _cors_origin - - def get_access_control_allow_headers(self): - """ - Returns Access-Control-Allow-Headers of this TabPy service. - """ - _cors_headers = "" - try: - _cors_headers = self._get_config_value( - "Service Info", "Access-Control-Allow-Headers" - ) - except Exception: - pass - return _cors_headers - - def get_access_control_allow_methods(self): - """ - Returns Access-Control-Allow-Methods of this TabPy service. - """ - _cors_methods = "" - try: - _cors_methods = self._get_config_value( - "Service Info", "Access-Control-Allow-Methods" - ) - except Exception: - pass - return _cors_methods - - def _set_revision_number(self, revision_number): - """ - Set the revision number of this TabPy service. - """ - if not isinstance(revision_number, int): - raise ValueError("revision number must be an int.") - try: - self._set_config_value( - _META_SECTION_NAME, "Revision Number", revision_number - ) - except Exception as e: - logger.error(f"Unable to set revision number: {e}") - - def _remove_config_option( - self, - section_name, - option_name, - logger=logging.getLogger(__name__), - _update_revision=True, - ): - if not self.config: - raise ValueError("State configuration not yet loaded.") - self.config.remove_option(section_name, option_name) - # update revision number - if _update_revision: - self._increase_revision_number() - self._write_state(logger=logger) - - def _has_config_value(self, section_name, option_name): - if not self.config: - raise ValueError("State configuration not yet loaded.") - return self.config.has_option(section_name, option_name) - - def _increase_revision_number(self): - if not self.config: - raise ValueError("State configuration not yet loaded.") - cur_rev = int(self.config.get(_META_SECTION_NAME, "Revision Number")) - self.config.set(_META_SECTION_NAME, "Revision Number", str(cur_rev + 1)) - - def _set_config_value( - self, - section_name, - option_name, - option_value, - logger=logging.getLogger(__name__), - _update_revision=True, - ): - if not self.config: - raise ValueError("State configuration not yet loaded.") - - if not self.config.has_section(section_name): - logger.log(logging.DEBUG, f"Adding config section {section_name}") - self.config.add_section(section_name) - - self.config.set(section_name, option_name, option_value) - # update revision number - if _update_revision: - self._increase_revision_number() - self._write_state(logger=logger) - - def _get_config_items(self, section_name): - if not self.config: - raise ValueError("State configuration not yet loaded.") - return self.config.items(section_name) - - def _get_config_value( - self, section_name, option_name, optional=False, default_value=None - ): - logger.log( - logging.DEBUG, - f"Loading option '{option_name}' from section [{section_name}]...") - - if not self.config: - msg = "State configuration not yet loaded." - logging.log(msg) - raise ValueError(msg) - - res = None - if not option_name: - res = self.config.options(section_name) - elif self.config.has_option(section_name, option_name): - res = self.config.get(section_name, option_name) - elif optional: - res = default_value - else: - raise ValueError( - f"Cannot find option name {option_name} " - f"under section {section_name}" - ) - - logger.log(logging.DEBUG, f"Returning value '{res}'") - return res - - def _write_state(self, logger=logging.getLogger(__name__)): - """ - Write state (ConfigParser) to Consul - """ - logger.log(logging.INFO, "Writing state to config") - write_state_config(self.config, self.settings, logger=logger) diff --git a/tabpy/server/management/util.py b/tabpy/server/management/util.py deleted file mode 100644 index 8631530f..00000000 --- a/tabpy/server/management/util.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging -import os - -try: - from ConfigParser import ConfigParser as _ConfigParser -except ImportError: - from configparser import ConfigParser as _ConfigParser -from tabpy.server.app.ConfigParameters import ConfigParameters -from tabpy.server.app.SettingsParameters import SettingsParameters - - -def write_state_config(state, settings, logger=logging.getLogger(__name__)): - if SettingsParameters.StateFilePath in settings: - state_path = settings[SettingsParameters.StateFilePath] - else: - msg = f"{ConfigParameters.TABPY_STATE_PATH} is not set" - logger.log(logging.CRITICAL, msg) - raise ValueError(msg) - - logger.log(logging.DEBUG, f"State path is {state_path}") - state_key = os.path.join(state_path, "state.ini") - tmp_state_file = state_key - - with open(tmp_state_file, "w") as f: - state.write(f) - - -def _get_state_from_file(state_path, logger=logging.getLogger(__name__)): - state_key = os.path.join(state_path, "state.ini") - tmp_state_file = state_key - - if not os.path.exists(tmp_state_file): - msg = f"Missing config file at {tmp_state_file}" - logger.log(logging.CRITICAL, msg) - raise ValueError(msg) - - config = _ConfigParser(allow_no_value=True) - config.optionxform = str - config.read(tmp_state_file) - - if not config.has_section("Service Info"): - msg = "Config error: Expected [Service Info] section in " f"{tmp_state_file}" - logger.log(logging.CRITICAL, msg) - raise ValueError(msg) - - return config diff --git a/tabpy/server/psws/__init__.py b/tabpy/server/psws/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tabpy/server/psws/callbacks.py b/tabpy/server/psws/callbacks.py deleted file mode 100644 index 22290b03..00000000 --- a/tabpy/server/psws/callbacks.py +++ /dev/null @@ -1,205 +0,0 @@ -import logging -from tabpy.server.app.SettingsParameters import SettingsParameters -from tabpy.server.common.messages import ( - LoadObject, - DeleteObjects, - ListObjects, - ObjectList, -) -from tabpy.server.common.endpoint_file_mgr import cleanup_endpoint_files -from tabpy.server.common.util import format_exception -from tabpy.server.management.state import TabPyState, get_query_object_path -from tabpy.server.management import util -from time import sleep -from tornado import gen - - -logger = logging.getLogger(__name__) - - -def wait_for_endpoint_loaded(python_service, object_uri): - """ - This method waits for the object to be loaded. - """ - logger.info("Waiting for object to be loaded...") - while True: - msg = ListObjects() - list_object_msg = python_service.manage_request(msg) - if not isinstance(list_object_msg, ObjectList): - logger.error(f"Error loading endpoint {object_uri}: {list_object_msg}") - return - - for (uri, info) in list_object_msg.objects.items(): - if uri == object_uri: - if info["status"] != "LoadInProgress": - logger.info(f'Object load status: {info["status"]}') - return - - sleep(0.1) - - -@gen.coroutine -def init_ps_server(settings, tabpy_state): - logger.info("Initializing TabPy Server...") - existing_pos = tabpy_state.get_endpoints() - for (object_name, obj_info) in existing_pos.items(): - try: - object_version = obj_info["version"] - get_query_object_path( - settings[SettingsParameters.StateFilePath], object_name, object_version - ) - except Exception as e: - logger.error( - f"Exception encounted when downloading object: {object_name}" - f", error: {e}" - ) - - -@gen.coroutine -def init_model_evaluator(settings, tabpy_state, python_service): - """ - This will go through all models that the service currently have and - initialize them. - """ - logger.info("Initializing models...") - - existing_pos = tabpy_state.get_endpoints() - - for (object_name, obj_info) in existing_pos.items(): - object_version = obj_info["version"] - object_type = obj_info["type"] - object_path = get_query_object_path( - settings[SettingsParameters.StateFilePath], object_name, object_version - ) - - logger.info( - f"Load endpoint: {object_name}, " - f"version: {object_version}, " - f"type: {object_type}" - ) - if object_type == "alias": - msg = LoadObject( - object_name, obj_info["target"], object_version, False, "alias" - ) - else: - local_path = object_path - msg = LoadObject( - object_name, local_path, object_version, False, object_type - ) - python_service.manage_request(msg) - - -def _get_latest_service_state(settings, tabpy_state, new_ps_state, python_service): - """ - Update the endpoints from the latest remote state file. - - Returns - -------- - (has_changes, endpoint_diff): - has_changes: True or False - endpoint_diff: Summary of what has changed, one entry for each changes - """ - # Shortcut when nothing is changed - changes = {"endpoints": {}} - - # update endpoints - new_endpoints = new_ps_state.get_endpoints() - diff = {} - current_endpoints = python_service.ps.query_objects - for (endpoint_name, endpoint_info) in new_endpoints.items(): - existing_endpoint = current_endpoints.get(endpoint_name) - if (existing_endpoint is None) or endpoint_info["version"] != existing_endpoint[ - "version" - ]: - # Either a new endpoint or new endpoint version - path_to_new_version = get_query_object_path( - settings[SettingsParameters.StateFilePath], - endpoint_name, - endpoint_info["version"], - ) - endpoint_type = endpoint_info.get("type", "model") - diff[endpoint_name] = ( - endpoint_type, - endpoint_info["version"], - path_to_new_version, - ) - - # add removed models too - for (endpoint_name, endpoint_info) in current_endpoints.items(): - if endpoint_name not in new_endpoints.keys(): - endpoint_type = current_endpoints[endpoint_name].get("type", "model") - diff[endpoint_name] = (endpoint_type, None, None) - - if diff: - changes["endpoints"] = diff - - return (True, changes) - - -@gen.coroutine -def on_state_change( - settings, tabpy_state, python_service, logger=logging.getLogger(__name__) -): - try: - logger.log(logging.INFO, "Loading state from state file") - config = util._get_state_from_file( - settings[SettingsParameters.StateFilePath], logger=logger - ) - new_ps_state = TabPyState(config=config, settings=settings) - - (has_changes, changes) = _get_latest_service_state( - settings, tabpy_state, new_ps_state, python_service - ) - if not has_changes: - logger.info("Nothing changed, return.") - return - - new_endpoints = new_ps_state.get_endpoints() - for object_name in changes["endpoints"]: - (object_type, object_version, object_path) = changes["endpoints"][ - object_name - ] - - if not object_path and not object_version: # removal - logger.info(f"Removing object: URI={object_name}") - - python_service.manage_request(DeleteObjects([object_name])) - - cleanup_endpoint_files( - object_name, settings[SettingsParameters.UploadDir], logger=logger - ) - - else: - endpoint_info = new_endpoints[object_name] - is_update = object_version > 1 - if object_type == "alias": - msg = LoadObject( - object_name, - endpoint_info["target"], - object_version, - is_update, - "alias", - ) - else: - local_path = object_path - msg = LoadObject( - object_name, local_path, object_version, is_update, object_type - ) - - python_service.manage_request(msg) - wait_for_endpoint_loaded(python_service, object_name) - - # cleanup old version of endpoint files - if object_version > 2: - cleanup_endpoint_files( - object_name, - settings[SettingsParameters.UploadDir], - logger=logger, - retain_versions=[object_version, object_version - 1], - ) - - except Exception as e: - err_msg = format_exception(e, "on_state_change") - logger.log( - logging.ERROR, f"Error submitting update model request: error={err_msg}" - ) diff --git a/tabpy/server/psws/python_service.py b/tabpy/server/psws/python_service.py deleted file mode 100644 index 877dc051..00000000 --- a/tabpy/server/psws/python_service.py +++ /dev/null @@ -1,275 +0,0 @@ -import concurrent.futures -import logging -from tabpy.server.common.util import format_exception -from tabpy.server.common.messages import ( - LoadObject, - DeleteObjects, - FlushObjects, - CountObjects, - ListObjects, - UnknownMessage, - LoadFailed, - ObjectsDeleted, - ObjectsFlushed, - QueryFailed, - QuerySuccessful, - UnknownURI, - DownloadSkipped, - LoadInProgress, - ObjectCount, - ObjectList, -) -from tabpy.tools.query_object import QueryObject - - -logger = logging.getLogger(__name__) - - -class PythonServiceHandler: - """ - A wrapper around PythonService object that receives requests and calls the - corresponding methods. - """ - - def __init__(self, ps): - self.ps = ps - - def manage_request(self, msg): - try: - logger.debug(f"Received request {type(msg).__name__}") - if isinstance(msg, LoadObject): - response = self.ps.load_object(*msg) - elif isinstance(msg, DeleteObjects): - response = self.ps.delete_objects(msg.uris) - elif isinstance(msg, FlushObjects): - response = self.ps.flush_objects() - elif isinstance(msg, CountObjects): - response = self.ps.count_objects() - elif isinstance(msg, ListObjects): - response = self.ps.list_objects() - else: - response = UnknownMessage(msg) - - logger.debug(f"Returning response {response}") - return response - except Exception as e: - logger.exception(e) - msg = e - if hasattr(e, "message"): - msg = e.message - logger.error(f"Error processing request: {msg}") - return UnknownMessage(msg) - - -class PythonService: - """ - This class is a simple wrapper maintaining loaded query objects from - the current TabPy instance. `query_objects` is a dictionary that - maps query object URI to query objects - - The query_objects schema is as follow: - - {'version': , - 'last_error':, - 'endpoint_obj':, - 'type':, - 'status':} - - """ - - def __init__(self, query_objects=None): - - self.EXECUTOR = concurrent.futures.ThreadPoolExecutor(max_workers=1) - self.query_objects = query_objects or {} - - def _load_object( - self, object_uri, object_url, object_version, is_update, object_type - ): - try: - logger.info( - f"Loading object:, URI={object_uri}, " - f"URL={object_url}, version={object_version}, " - f"is_updated={is_update}" - ) - if object_type == "model": - po = QueryObject.load(object_url) - elif object_type == "alias": - po = object_url - else: - raise RuntimeError(f"Unknown object type: {object_type}") - - self.query_objects[object_uri] = { - "version": object_version, - "type": object_type, - "endpoint_obj": po, - "status": "LoadSuccessful", - "last_error": None, - } - except Exception as e: - logger.exception(e) - logger.error( - f"Unable to load QueryObject: path={object_url}, " f"error={str(e)}" - ) - - self.query_objects[object_uri] = { - "version": object_version, - "type": object_type, - "endpoint_obj": None, - "status": "LoadFailed", - "last_error": f"Load failed: {str(e)}", - } - - def load_object( - self, object_uri, object_url, object_version, is_update, object_type - ): - try: - obj_info = self.query_objects.get(object_uri) - if ( - obj_info - and obj_info["endpoint_obj"] - and (obj_info["version"] >= object_version) - ): - logger.info("Received load message for object already loaded") - - return DownloadSkipped( - object_uri, - obj_info["version"], - "Object with greater " "or equal version already loaded", - ) - else: - if object_uri not in self.query_objects: - self.query_objects[object_uri] = { - "version": object_version, - "type": object_type, - "endpoint_obj": None, - "status": "LoadInProgress", - "last_error": None, - } - else: - self.query_objects[object_uri]["status"] = "LoadInProgress" - - self.EXECUTOR.submit( - self._load_object, - object_uri, - object_url, - object_version, - is_update, - object_type, - ) - - return LoadInProgress( - object_uri, object_url, object_version, is_update, object_type - ) - except Exception as e: - logger.exception(e) - logger.error( - f"Unable to load QueryObject: path={object_url}, " f"error={str(e)}" - ) - - self.query_objects[object_uri] = { - "version": object_version, - "type": object_type, - "endpoint_obj": None, - "status": "LoadFailed", - "last_error": str(e), - } - - return LoadFailed(object_uri, object_version, str(e)) - - def delete_objects(self, object_uris): - """Delete one or more objects from the query_objects map""" - if isinstance(object_uris, list): - deleted = [] - for uri in object_uris: - deleted.extend(self.delete_objects(uri).uris) - return ObjectsDeleted(deleted) - elif isinstance(object_uris, str): - deleted_obj = self.query_objects.pop(object_uris, None) - if deleted_obj: - return ObjectsDeleted([object_uris]) - else: - logger.warning( - f"Received message to delete query object " - f"that doesn't exist: " - f"object_uris={object_uris}" - ) - return ObjectsDeleted([]) - else: - logger.error( - f"Unexpected input to delete objects: input={object_uris}, " - f'info="Input should be list or str. ' - f'Type: {type(object_uris)}"' - ) - return ObjectsDeleted([]) - - def flush_objects(self): - """Flush objects from the query_objects map""" - logger.debug("Flushing query objects") - n = len(self.query_objects) - self.query_objects.clear() - return ObjectsFlushed(n, 0) - - def count_objects(self): - """Count the number of Loaded QueryObjects stored in memory""" - count = 0 - for uri, po in self.query_objects.items(): - if po["endpoint_obj"] is not None: - count += 1 - return ObjectCount(count) - - def list_objects(self): - """List the objects as (URI, version) pairs""" - - objects = {} - for (uri, obj_info) in self.query_objects.items(): - objects[uri] = { - "version": obj_info["version"], - "type": obj_info["type"], - "status": obj_info["status"], - "reason": obj_info["last_error"], - } - - return ObjectList(objects) - - def query(self, object_uri, params, uid): - """Execute a QueryObject query""" - logger.debug(f"Querying Python service {object_uri}...") - try: - if not isinstance(params, dict) and not isinstance(params, list): - return QueryFailed( - uri=object_uri, - error=( - "Query parameter needs to be a dictionary or a list" - f". Given value is of type {type(params)}" - ), - ) - - obj_info = self.query_objects.get(object_uri) - logger.debug(f"Found object {obj_info}") - if obj_info: - pred_obj = obj_info["endpoint_obj"] - version = obj_info["version"] - - if not pred_obj: - return QueryFailed( - uri=object_uri, - error=( - "There is no query object associated to the " - f"endpoint: {object_uri}" - ), - ) - - logger.debug(f"Querying endpoint with params ({params})...") - if isinstance(params, dict): - result = pred_obj.query(**params) - else: - result = pred_obj.query(*params) - - return QuerySuccessful(object_uri, version, result) - else: - return UnknownURI(object_uri) - except Exception as e: - logger.exception(e) - err_msg = format_exception(e, "/query") - logger.error(err_msg) - return QueryFailed(uri=object_uri, error=err_msg) diff --git a/tabpy/server/state.ini.template b/tabpy/server/state.ini.template deleted file mode 100644 index 8999537f..00000000 --- a/tabpy/server/state.ini.template +++ /dev/null @@ -1,15 +0,0 @@ -[Service Info] -Name = TabPy Server -Description = -Creation Time = 0 -Access-Control-Allow-Origin = -Access-Control-Allow-Headers = -Access-Control-Allow-Methods = - -[Query Objects Service Versions] - -[Query Objects Docstrings] - -[Meta] -Revision Number = 1 - diff --git a/tabpy/server/static/index.html b/tabpy/server/static/index.html deleted file mode 100644 index 52784f13..00000000 --- a/tabpy/server/static/index.html +++ /dev/null @@ -1,72 +0,0 @@ - - - - - - - - -Tableau Python Server - - - - - - - -

TabPy Server Info:

-

- -

Deployed Models:

-

- -

Useful links:

- - - - diff --git a/tabpy/server/static/tableau.png b/tabpy/server/static/tableau.png deleted file mode 100644 index 72d41cd49d18ebd425595d65fb7d6b3bf4352526..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33575 zcmeFaby$?$_BYP+2pFJ>-{1BA>0H-v@44@_>$BHdd+oK>tpSSi68G;B+(SY_x-TUus*Hqm3m+(F z-a!TaYN^qj1DCsYl3I>PNN8Ay?;l8U2>=B*1w>U7swpSSYiw)HXlP<<1ZH%#wgae< zkobjM?F@}A!BA2ouo=WgfNZ?@&XN5gQ13`uGUsIj=Zh{WY_KT0%gQ)CNk1%5~!sB*$YI6q?&Syq$0KsU{Vf7 zP6lIU7IsoD9!3@pPA(oUdQw*4Hxn~A6DuBZ8!s~}>CX?D;60$k?_gret1K%1 zvpe9I0GT-yYRAjOtM#j!o$PE#LUXX%E|yxFgUu|Kn-0PY#hmdH1dyj zM8S^64iGyi#MXus(XOG9trJv$j11Ay-=ClJvbOuXBOAw`*a0Xqxf3^*0sOn}1W>N+_+B!KHgTZ+2wZ9S znf|@fKUe&p?SVxNp#O(ZUb1e``R_)&yeecB27K z1{PLC11uc8%$zsdxIy{vZ2;n9VhA<-KWgPB?TvO!jCoCM9jpzZf)Hy%Gcc2#jTt}F zKX2Zk{F|wGMQp8X9e`nh1=;wS{?qLnwXPYBSIWi_YG`8&mJ$^NZZJY1CcHfCYyfhs zJPfAB#@r0-tj4SihRncEFb|tCH>W9&DVT-(8bmp{o7MkE)1tP93X(AGqk!vgBbn)Z5RKcng7w1B*YP5Pq!Ov3o!4sDpLX5|LfgblauKlzU}wfzFCn@~@Efk1X#BGL8?IkM zbd%sWTsP79W%)N;zl7)}!Ed;3qVdb}Z@7L5(M^KiaNR`Xm*wAZ{Su;^1i#_BiN-I> zzv22NL^lb3!*vslUzUHv^-G9u68whiCK|si|Ay%aQzaZ zn*_h%x{1aw%fI3JB}6v~e#3PWjbE03!}UvuZW8>4>n0k%EdQ@?-TUXOe_$KneLolA z#l9Y6{RhBHfuzQg%5q3Z?vzMKfB7OIom>IeRU{-QW+bFF5E2saTO=er+X#aeaU>*Z zFDcO%s;=W3^_GgN$4SSQzzb+jxtV!-`$iVDi+Mwn&b0n=p4~Y-I_AuWs`o7!-t6;p z*l)Y05GJc8r9-AXy6+16_G*xkjnP8W#8;OdZM-+7lWaDTK)xIKoQoD!R@Er(h!KyY zRrd&N+MP4wwrgN{Xkmn%ffKe3S9p2f`Lj{}&{fZsi_UXVG{g&yz=ss~8|ifcC?Z~v zM0~E_kwg?=e<1z>pNHr_u1LtYeioX3UY`T={PFn*@n4{yH~wx9>3*(6>CHH)~7RI<6f$YXZk21KgRtR=r0MQ zk?;bIz51_F+fx!K<%o;_&w&FY_+Lth$^0D9|6T&7PaKy>0Hl+ncTnH@fr`GM z=3NykO!C|kD)fpuAQ?eTy67x_X?Ep#0qytki&7F;Z?{e)}G$CsCvCR$Cvr zYq(5T8v?EWM7-__Zo92lxW(Onveu0eQg7k4{9a(DHWYh$=dz)s;$k$ea-#eyYd;z> z?wWPU5jwsQXhY5XXXC;xUWbinJ&BO|J#P>xTa3_E#%|%2^Y^_`HnYO%Z~lcLf{vFR z!D7pWK@0>9yqDG-$ zd_M7$eui5e?PQXj7fqQKu<!Q(Cq!BP2PBCE9V=IBMh>#%_#UAPgfqG1pSSCxbZ8`4rY%TC zGD5oz9_zevynD?}0BE?Adn+E#ELax~s8IyGX13Y77kW4*P&d^!1R|1kw5YNq-K63f zU;mU~?f{15GfAa43tjeJehgj5Df zeI`O&7W-6Te|TUs4tn}jF_tY-Xe5L@v9QDFh3)saocGQKEwY#V@!+N}V;hw7I{K^) zEhe9<#%}*m9l$tVPY`?cFoY)AcxYl9*FE#KsTnX9Y(G;X!v`2&b!AFnV}85SOqDrK zsEf;{AZxM%l~tWH%-&JpG1o}MY(6iqiyL-i-8CCuuVIi|e{JIty_!mzjT`IA(ox2N zwVf$BzI34Ki%mh=yh9c5O%Jv>vU2+?On7n0OLQE1Sh&t=#$=^)WXdBS}>{9jRb~ z^b8H3JFep=UlP}IT8>AdwIwn(X%Mp>%FToRi z_y&Z8w1mS1h?HC3i-d0!`X!iDqeG#Ho=nnk#GJgJ&> z>dn4m*})Z$Rso1WrNC*Dy@LBOLp3}~c_BYMcuU8|tLJMRUQOoSqpiAkL)WQ62at^T|AdFaZ(K*Vl-&FN$Z?+ld1rXVAs!@VOvFLIs9Sw$W! z&@y)tY&$eGUx$j+6>vw~P%n)Ue)qM!9i~idiAvo!#eocUoCaTn(7{;nEUBLH(~rOJ zDy|aBdlO%tJ~Z$)^S#GpWIrMJ7X(W>k;#4tNAV{&FFG9_XNgwMs6!>oErjoV|JH;Dqs?6oFe>X^B5IJ*?6$7WK#LsjHGuehduRS5V_Dj5vmk`UVJUaSXoEeJ zS7SIGlp>#9=_piMXu2chOsRPlrIs-+#}VJ1u^Xks7P*J}*AH)%STGAuVa znIz98@R)sID%t3n(TwM@LAU<9Z#}>k!A_JHobQ}brITM9CN}S@Vsm#KAgBUC#y^(!BeqIDFei2z6?%rXiOVVcvMfUtM~|1v z9)_-PF*fx0b%Y_}e9%lmq&k87W5T5)1fuL4tfuV`iT20A{SW~#;1Wa7lQoC7j? z8U4B;lj+m^Lyn!^gT%*Dvy_CputZlSn_60JwZ5?1?SO8$W~h&Yfh}w`(Ts}W?r`)| z6{N_z@~G{zqK}HJX&gAUgTl5M6~^D!R@k`3LrL^YUPraSPnAlOz&OTPhzrW@Rsc+y(}o#;sc8b*BjR?ASHi2*8IrAI zbjG6`K7X!=KEv<$Q(Oz#;_d@I`PSk;F_`f2BB_#Pa_hvqd4BAOO@Og`UHb78u{Yj9 z1@^ALe+FspWT}vxR~FVcPMtdCk6i#?FEf{mj*q+loubw+mTB9oQc~ zwpYZ?n%V3^q(9$`DiDmvltM_@Z(iqmSC2_PtvFSc0JxCwnq_`}f`IBF`V+cq#q!6} zd(e5WRxF!#f_PdBguOs9N*s&ufF1ZbL*ffG32gM&*doSf9O&uOW%i_!IY&{{xQsj4 z20NqQ+@9c~ca=jJApF}H00zZUj-A6;xGTQPw0NgA;*E)wChXq_OUwyVsx{Tcs_ylE z@WU?nVL$wU2T%?-fBqQs4~+hSU>n*`I|QiR#ne>gbT-i_=rVhnT56y_-{a%v)mM$C zf#-F(SY8z{F>{HR5c|&&&Ij0UuG`{<=#re(hWG6@o~^Yu&luG1PYK^?ox}=?Il>x4 zjf@Yntc=`s;#zE2dYYCKr#Y&e6Q0|iH+`Y%CU zN=;urq$KcjEyh`V30uw_avOr~+_||@{4bn|c#foJx4aN1rFE|}$VfQjoz^GVLB5+_ zW2mL^=%!qc4O}9h_K2XC`ad9kfS9O4)XM?9AaJz0GDlae^*W3L|_)@FIPa^`!kk-%kI; zlt=eI)U{hNasV+WIDpT$dWa7$mW9*cFp=1?LyD`$k05Tb6Mk1yiWB$_w*~1R$mE=` zzC-DTSTQDG)|m|WJ{$W4joXEK`+&YAI5aL2Tox)!bbSEBOgZk_nFADb`mN*#43-xr z&n=*sD&_@QvRc^98yyKLJ!y0a*C=rN{e&h@To-YjFpbxgO9|^5(loMV=S00Ih({G|E_K1=+*1;!(7JKYviN zK0m{lP^`w{ur zNJ&S3FMGWAHlWWu0sZV?x8mS1^DZ3wnPhm+nZFU8SY_kY)ulFOQF7{b*J6}}+tw|j ztQfWJlTqFJtO4E4Z5D&%@1{=?LT2_ZABMRf&jgfrUwsEo3nd14^#f0sV)VZu-(IDl zvWMw;4-D7Yqp1gWmgS;}1wVgPGQ7#_`E9fvELuj|rabc{K>kq9;G!W*6HtA@u1$}5 zJ?w*Ks@6r|f-aUARAZmu6nrgr<@*CNn$gsq`H}hfyi@7*0o|Ju1*k{+h)CBa;*Kv< z!+JCQvA3GuPI+Py&!bf|80^9qaeQT|%K-a*&1Iy6nySR{sbJ}X@bLKfXNESxc)a7o z1~zh3Ia5|4RTZ+g=o~aZ-1b7VmW(v25|Y3&)leacaP$BZ$^uXPj~o2RBXp>R)-I@ zGkF)c)HZo+^bH|9jz&_h16xAa2zk2JGseVN_;pdK2BrBD07s0skm#I`zQIdqzS}L; z*4u)Mlp4*Q^QyiWjd6~Wu;p$yTIX>6RGh`+krZH`;SD61hMx?IGb5XsurQ2CQt<=; zHQ|i?cNv{twzr4Upiw!!pwy<3$K{hR!=j4r6*A%Z1dp}l*K(l_1HjTtAbPy0p-FIF zXM#qv%uQ8IHUg)Q92a9ftL1yAnG3gwb&$PgQVAh}Lg zm$ffO3NQkn{(ubNI}Eq_jVS`X7=dQzk#bhj##xh?Tw%TM(@ z#=@%&63$CnmX|4YB%>iGE=w8x)Ka~fk`NSf2d^meRFQ5HF0HpJm_dHeD^je;AP$|H4@X63?g+RR zYXzwh7RR+1weo-4_(Gk$w;uRX^5PX;Nhf1bm_XK69wa2*^p7V27FDUC#ovPTvCWe$ z0Vp8k{0Ys@$U}3^JPhKsUlyo{%;d2yl*pgL;0$@9AL{NfP|BD2VL;IteL?(gmu}W_ zbX23CEt;kDDO#n3-W_49wWXg05V{9rm{KELb25yNTeXxfqe#z6sZpT)b)nYfI&Sewe>6%*@>< zwfk=4yMtB+y{&F{{m5l@-5s+C7L98;pnc5p^=vE({KwXYJoPJ3TGv$zOX1KaQi{pYV5z zV2dJ+5}#6mrph%ijhaZ=<&1n@pl+?d%#Z9?P^F=30gEfcm$U=1v&3uJb3k!EZML zvBvZnY^2J-vrt)#TGpfZvK*!<45+^Pt#P3&3)T)7FU^lK87V$rl*w=z4=pD^sH%@d8T>!meNUsudpl1y+2v`Y`D8ORM>HV%EHTvm3IaZs;$o{~|CXdo`A83i${bMV%Rt0=%l_Rkn$mK9f37kn-^uo8GrCPqH_@Q`}jh(k%r65xY5qGXwrN|4vQeVEc*HMIzHEcZgdR6)f*$3h`gt9s-Mn;Y5c8|k&d1S~+6 zS#RPBtvc!j`7GsNyD6wKW53YWTd-8MV2+Zj#RH78iK=yS8HsH-0VZBymB$_Tn%-Js z1_7q~J(os_*R{ue?lb!CwHM!m#F;N23GPms+iUlHXMfBfzy5ftXd!ZUjxt#mJEIRc zS4~k%rv(@Z1mj#7$g&3Zo0oCtFSnu|U2w_tyW3!2mAF`mh#&hV`Q%X_Mh46U*({n7 zl@^8iJmxvvkbP1)eDprbaxbh&RSq~?IPCF9#V;q7Vxns?r{bDL;*mN&o)VFF^p2qS z&Y4SBev|K9AD*xm-DmilSp${q04(q_f!@N`0_@yz+AZ&L04z+XL@hqfLM7>&w0g@FZIiDNIUU3&3z#2Ij*MFl7VaL{MKI)*K@;I((~erq-JwAY z(wAy2pr#lR9EmfsCybE7sOaL8#Z&dV+TR*v4aVCEBxw)Z0Q=D}Ly4)R&YVjEB3F2g`O= z$Z1)&-$^<@Jiw09D8K$%<=GTl<=UuS={0OBzBs86LD?w2g<>1UsZOwGK~2^3l?>k$ zV{xH+^}eOoV&=iEB0K+&d@KyH9tg^&BS%HdC&) zqtzW2k@|i?knNA6+*OnW^92&~1I_R2Zq;AIG2 zkZaQmKq2B-c{R@MH77QDMUAYSZ5TcIe{d7hNENai#6snPkuaA-zl{Qxdt; z3ON19I6dmDxAKn(Baok)DB}0Zl%G- zmu?q@4v*B!{3Dj-B+CrM8clY$yuMu-G_T^{rrZi^#u5`n$ zN0?}f^XB$7^mk>CO6y`mb=>biwfI&N}t*C zS&zFH0q$BK6yI3?6|sAAGHKrW6<9jbfZ4}BACt6-i(ycGc-X@_Y_+;18Z(`?l;7h^ zfUZv)+&<$CXiajas8`$p41;s8zs%6e$K0<#0z<5b>OZ^R5f7T+QSdxtV=QIQlTRBN z#%gzH>%XH198`+RoX=(@XGh&Z^$d+4PvHGrr+2hYUB%Nsj4AZzR#WvJ?c zau6lJWN|Jd@!eD#GL8emrnGwcQcAnUQmDu;>8CiHd*cgzx%=7V#9BA5`dOunYF~~s{JS);DVh{TL zRg`6JILarY09kVuk1rr+i2(nng?>SWmal{{qxg?>KkVnWg9^yfXsK+DIIto^gC`zn zlj85IFh@=GWgmq_h)jv8P6!1ux_i5|d_?Tx#wVz`STZ#Fc3;t&zKuQ!v+L6KESMT= zMLP;3uw~E%su_2?tG|-iZ|Y~uZ*1S`bB>Fy0xX& zHHBuh6B=m=685l2F(eKVf_c{bW_UrbRpP-^pND=A+Hlx-oVJ~E9mrT@r;N>h)a^;> zQ>)fbr|yHGt^`mQUedV1#wG@zffBFqRI(}U)z6+tot%$hq%zRR~Ki1Qi7Xfi;+DX-}V3nMm@p|U*~IO z$bpt>D)@vbA&YF@13RzdZMRy#5dMs$k?QH!=b``VhQG1$S?0Ed&@b=rrtA?5O-78_@+Ej z*a2~|$fA@lQfV5l2pn^*+p8>BD=|6@R`NyVar1r3s1g%oWtlyzIMXp&?wM~Zd>`T2 z+|7pdP49NN&rAmb(tj4Qy{~~g$f;2`JE5Vaxt_)48ECb^q|x9k1cY@p7ChqVTu68b zjGMR*H(c$}$?zSYi9yo|9pf|H4LqKQP7fR@$ zt(F&9rvrHRWFJ<1!1Uv=NKf$#vZTu5uWqk$=F4YQlW1dfWGVwFyWcd4C*b^P3?{Gq19BPwpQsh#?2dVCU;ZnI|KgeiSYEFA>>rmumMZb!FAEkjJN5rtnp2WFRETpY~^{rGuqI38so(iF45hXx(p(HPH9U%(xIeYWn=5*#Ans91Dqgjl4eg9`_P1}dkz@IrFLDqR#H+DU zJvt%CB|75@CY5Ek7e{!)r?9w>Lmvj6P~qPWP3lso67Kxmh%2Se*(LMlAaohogL~Mm z6k)oM*I&sjiv>o>(^JYjyf(h5S+&fq)e4kK|E3}3XiU^7{RVGCwJ0OwPZ(y}+*ZEz zVgiUt5S~G|D;%J0n_}JaWIeeN)R_{<9Z;XdC__w|)h%PEC=H@ZUA9!aN% zIUR?6$#U)0?^^6HU&m+>8rss3LHO_<290LmmRM%0Zq{XD>C%_Hf52DP?_a-c@Mdvc z9dNb{2Uxw;tpmZ`1*}#4ZnepO%{5o+>>C)v;1%L&e{r=e6Mtsxouox@E*TC3+z3lK zpys&RHs?aaz5q&x{AQ-bs7z?w==&@o`x}Qk>`~h_dk!gaaL_y!hN|@ z7XAc$*c5jv48i@ZR3nWm3ty5w^8zAu%RcRe)bJFiW_Jq_sRKx@A$(}^)k*X*9i)x9 z0*R$-<6}$PT3pd68-#$Z^3xS&JiK?o0*-K@5R+cTcS`#ndmw;pJpD;3gqbN?SvND0 ztmW&2uO_;DC_S&1TIATh0@pO}dT2dVp5 zXxyK|{l+6kUoojMtR;5P&2}ER0!Wm7`K0uzp5;KCu7)*830YCGy z{RL4}b^xc*pG~gp^o@MFzyK2#64aKDbaButuV!S&TN?ViwW>{{wk!iZiv~MaC78(y zXTR?rrdq{)aoBh0<11aO+U@sM`A!_~De`l48y3tKBh2T@e2O`9F+5@)so=hjA*D9%uCkcl^_hd+90JVc)BE` z)66^gJ{b&?iJOsnA1lfYJ7I+)aUp7=ZCsM7idaXfq`z94YUDN$+^i&h7YEXP1(MZa z4v8lpC?!hYe3l`KKWc@paUIqv@?Uh%>V+yM`C{c8x}{Mxqjf(7Y#BJkI;DP*Kj3Fx z618-#7y_9TPK|L!KSTZz00=4F_bt#stMjHk*vf*9ucEbbQ7H(za52Ai$hv~=xx7@N zaq-QH*go=!Isd%#+RRj%xvUS~u2N%`3m;r2PqV-9#f$XtXJpvob7Z09hO+Q{*W%{c zImwW49}sfo4*QzvRQiWU2ZimO*1?+1m1k3RX2waU#@pE$j9YqZe@c1i5^LP|3t@=c zdbjmQlj{MB)ZkVH0Z}0KpJ}MZr-K`-?!jUh2^P{(B`;DE$G47L-r4mG-6=95m!}V( zIN@Ei(a8v>4l_YZBzn9|5(~;M6{g_amZa)=K1YF{Cs1WGGFg7N7PL zHpvc8T3UE>f*4}T4{=}|{c-BF88GX7oT_D7z4W@^Jo)B~fbZNxjr9&aZci{d)tE~* z2%7A&^(uj&6e2*-l{sHt(mRq~FGNd-T*t2(j<>;5&v^<2Mc+x|m7XXk-)VYl8i*|u zYLR(fQ(v|(c}xzh`^(Fy%XH|PjBMI-WmFU_HtM%Uul|(EfpVHzlOr1|Qw8U-4rfT% z6*nw7SbRw<3$=n^m=b=SXb0^Tt)L)cm>B!d)CVcH-6sYE#_9FiaopzdWU|wa>xf}2 zwJ5%L8!2DrhBLca`yi%4`~}d;lV|uB$*&`w$Shs*FYDrX@H)^RLpDP`S1q%4AogYG zkn*_ihj_y9{m5LB&wo7^V;+>uuhSd>j}58z=3gc`m9oH6hL25(PYJ9Ay6?$|bQJ_o zr^vub5{w`jmpz}~07`$WXJRw4aA9N<;00?&rflni(UYcP3A~_7mG&l4E&4}?gRyVr z|KN$(eLY+G1_b>76D^0{`qU||#ZiNjexA4Pq6>P)sdk1VeQ09dB=xQKLxiSfi7f>B znP&O@LHutJ%d%_Ko(?78e5qM$Y&y2*&4X-wbg%Vg$UrCHHg@87Qhryn-yT5O9H1`E zrS2rcY$THKt?vDUHwk5EJ#cWx!=4?)NzHyH1LcsRo@czV^eAp5GrGRY*=%ASHz>R8 zh1;xdxta5nr8?jTjIz`W4hcBQHpZDC2+x4KBy=3yK1YR7L0Q z14d)Fz9Wyi-P+M_%H<5RE<&+v+C94C~dujTRf=c`Ih(x2#cWz{I3-ti^ z8GEIK1yB0$NJt@0874Z%VqY$eY#}@mNFs}9Q%UVD0*-IMX|EP`-LNk zP0r3_vjv;=RtQu0m@Q=O6Ktb!2Z%@EI7 zjux;m?G2o31@sG1{qiBOe@aL#`T=?2+oY5d%+~|&c#kC|L-R0Mx=B>PD`}tbPOCy) zibHBbLl`7pqZALw$A&{cFWD^0$xy{~*q&yjL)!!JMhiCfuapImn%;WBia+&#)4(-L zz}I?S9P?RS4S)XCeY}EDpW25oOhR5DP@0?YS$FWgya$C?@aCXD5W>N1A=M6B9$z1( zS({(LHi>7Ce&#-;i;^4tGFdi&_D$&L* zH{jRO$(0l|v~GRqQ)feK{mNgWK+xCmF>JTFijX_;k))*AsuE&JBvpixfWve@VeUxb z0MtdPMysphC>$`a_A9f;_H7I8-({=`t7b`2eoOySwl5ZOp-j)05A158Svdm~J(7wJ zmuaX)#ciF`@}ti?;ehPpv)UPF-UG&U`>KQRg25n;gm0;7e!q z$Dn1iu>|w`bZDOPWu@+--3boM%r*pN_=Tm1-lN+;4vEf6CDhVT$HJCsjWv? zW58Benhz-Oiv%(Sq?A+dQn;@a`0~mhik7X7yWYZQ+^jUW#oXFRStB ztLm9JuYq4J5TVtH8+1VgM}U+9@yv&oHGa)#4yS+e*@_fP8-(44wR^2_jKKPp)rxv* z>Q-`=SV{vr3bNIUJC=3TA+k$>=a-}3ScwQ5GMCOQ2dDr?mnSOVtgd%CD7|{n^oBvr zBP4#PKr(KOe65G6YS2fh(R2P|WLVHXxpAqsiTCaYIbVw`Ho5z#e17tQovLeo8?!!~ z+vdS)HQ^EWGJ@}jTAO~aH5;nWOyMX}rW&VyQT8ETAb|qIuFJ@w85|t}mYFEqZX0Rc zrdLg~PvyK1Z|@>|3D!b@T;vDj5qUHr?7UrZd){EYEQHHH7%!vGb@vhCFw!^2etI<8 zCA;SbC{|N2R}KRMPQrpBUe36%@R4m6{o@Bfo8J?^MjLHgumdSmOYm+8mVAI3g$zN_K`V7G^E0cX#mq zRn3U+(r~qJa51tNui%gxJ%>Wr0@fxk#a(#nu;#`vM!lG^_fmo=*IlULa^B#wV{@C8 z!5cErK31WaBzu@7&jy%3b;`a^M<~Q=_-XnoSp)H`@D9XX6o9;wJt&F@-?|x)b~F~Q zlmw*3=m(0%mdg>&QDl`o#pcC}OR8$nAJ$F#l2jocHKmcTcait%Ud5A7x_CgiZ@xA} zuaFv|WL%;e6qDH>e4ie+D?)W`#w%9w?tbl2)*okozi?vV0_F*jtCf4Wxn{)q0@}ad z$#q|qi-^mBy9~Y4&sz5R_Ey26CSEe`Wup>4gp9ctoiD8JIEIoLj4f?iTSP%a?x124COu{nHRzEx3Bua| z*y<6}C{5@25qXae#Qc8V0Mmu#^SLPRw+RmJ7DU?88obZl&6cv<~$rOfG;^miZS17Rwj zWnJVnpB3$rM@_xX4$1=!3sJYK-wJl!J=(d~$#)6lTbz6>9_T#n(UnFd0-WAvdUo5N z@fi?lDz}PnG@H84cL0ww?P!5hBrzN4`d>>>E7i`L4-~sL39f4o2wKBlh@zEr@jalU z5v-xNG*Nn{ou<6_R_Ii<6+)8LdR{>xn%h_(_w*zSM|jjsQqJ&^3h&I*L8>n~ms$h+3AdeBR9g0EEn(VK(@kG zRVkkb<4%!>8VC@lns&c#*49}s*-+vO$<_R_sgi7<&tkXiPjYz=w)=fRE9oj$?pDGA z>6Fme;LB)xBdr6cr{?|8sF`v?V(u`-T$0-9y96-A!INoDGdB0>5WHUrHAwaAEPRiml8 z=K)_@o;*Y!mIPUbRwNcangY|vsq=i=crT{K31o_7gVxV*+5~xyBg6e1Q~JczE&Jdrqacc-@&8NWI|h_nFFI^d2M4dsN=~d@M1P-eOZp?5onPva&?ysO1Qkg*ggK z6MvH9O^9j77iiykO3?mLB_d79_abwoH-)C+zQEl@(?uFz# zbhm&zLeb>qY9bRJWIR|hfwe(!v!x&K=}2k;$HFp1T-q^(VQqi{U$pGoG109~*FK;T zp9D~rYZE1xL|BjI*BLpgXfZj9VR^lg2XYG>SmRDgk8IS$*j|9hgeJ!G1(_|hM|%<3 z54>r@5m{<7;UVU0Mw+z2QEGp6R<${-W|Fcw|O4lMmt?XCbFj<}S$FJ#4HtJ~a0%r^f!r}XfL+TSm zTJKxFTKIjrx|Mr0{U33~^Z3Fm@O7q~uoHyj*8|4wU2+V))DN^jCCN~yN!q(nL`zbQ zn_m@|UU?M|aZzrHoFtOf1{O$R5=o@v>*_1!)D|eUB*@DJ2D4f8Wo+13;wGlh?}vOg zYb2OoZob%|!9u5$V_eMOy3MUbOh(oD`q7Z0w}w-8hHB>OFcwi$|GirEZ&V5&mnh@o zXt_pA+aC78VENWKLh`pFZ+8HGzlv|seCZI9NGt5e?Cl|1x|vVzE0GEGjeJSKLMPL< zdp|$O3S==wd8AdiA<2X~OgrvX*z})0iF_y#)m3~S!#^QVUD`e`lP;?MfdepiNE0{9 z&Xf3s#1}6q1P%q_fsCKeHSSN+q+m}CMuE&Cxm;hI$hzc*2_Os!hD&6#LBjh~<21~! zcS|J7-XlUcK>pm5ZxnbfXq+4~8rF0)7+3?t8b!5fgy~J*83j@A92*8Dn zn3UIC{Oc>L>Ao7^Orv}$K2DI9{kkl|f?Aw_$h_z7qI+NhVPF|=+FDwaSQ%hhv*T@Z z=v_H89{L#Qd;66aI*TheiCPYlX}N@o9e5Se@rtmsjp=<4tp}QmEbR@Msqw4D{FPUGW`NRxEV612UTE zQ>=$gRU-vOC2xWkK&35`Am$*kO)PCBivi5RDc7dlE%&D|%&AWmc9xbrg%6BME~!s5 zmm)h8C7}d;Zz9?GN+bKTyrZ}nY8GJj9^8utoEV;%NrZg6+KA}vz}d?8(O8gfNShOa>JnzHk8{{vd!h;eDILFH;rGFEdkQ4>1VvkNR7Y7Y-Q6- zt`WxQseG6n6lF|MQo_inw-|&;1Me|K`$fby@@Q6-m)Cu8;dWvq`CvKkf;(J!v{a{F z;X_4t4)$${(vzU%5U2SP5{cGW-gmd0Ss$zV8!{i_4Y7bDyS=@=CWKqVOrrIK$3w`o z14SeSNIT*I?8A7#p2%?fQw9SHSu(Bi9?aA_@vzh;w6tKi%HxYy#qD6eEU0bBU|NP$ zk4&Ci_V}okFIVx9yXaKPo4nqa4lkvBv=@ZAc|pa85JjHoz?N}mfO7YOngAjtWbX2hpSRW2N6rqq9xeF#k6p02=_|i z2+fLH+hjH!-tn#?r?zxYmF_4MZo?S{ioq+J9fke+C3v`H9grPY1-SWuoY`pK zI*=KISG2s6w5YK4v2N7)A$VNvABf+&{ZlSrI(~@BqK{aU8-&_Tkvm3Xu_> z1!N;F@`ya$^b+;%F%C7exsy+CVMRMRu0HKxMNPO30geskC-Hk8 zaXA^#d@u`Kb(vX=--!%O;U5ANuN$7nHEG z=Oey|;2qBQQHjMFAg?dWt@uncmC)D)_vnGX*13PfEFQmHVkY9Hg%1~cOV)C2!_^l8 z&lFU$R=*X$+?Y<;nY(-{>JPjZZD)S)?y{+~UE<7-&Eg1JuMTi#wL9=C>iCRx0Wg+= zv4O*J|9XS^hxq7Ox+JxwJ2{hf@62GBIKaMHyJJ3c5(EUomJmD82f*ue{hpiN;Ajrqx$;u1ybE-}82cOdALR&Sr=0cb0Ggxx{Bc?w~57K$B@_18MJ}wY~BTYN&TwzMb57Q;-|ByMK^<8gN@iocEAQwDZ-TQh>fJ zv27-Wqh*Xv&4XqNE0F7WM zI|7LU*0ut&6F^ZA1;k1lDt#4r5F!vGu|U}Zlue>#aiLmd)3huFAz^uhB>@^9mVHU* z-ca8^eRIw|nK?Ie*ZKYC-kI;`cPGL5G5vX)?(xM!bWI{A?5F#IY^zv} zAL7(ha!!Y&aMvCDlf@yZUbU( z_el0RC86ap-N6(>DBTF0^(KOX448fm zlrx?xHqx!}_0wCySVSrA+GTBO^b5lrtMsnUkxv%%m-=8C&!VlV#P3VmD%ocjHe|o^ ztRzNZ1K6vt=hO?MQy+ca`e1bq>qE0+Zic5)%z!R(Mg1j*olO0s8bNjlPY-Gmo~|$R4|acL6QgsVnOOS<+^zKiL*B? z$&sng?=0d)5MKs$^Pho5Q=K51rd;4T$dQ|~aY#of zT!X*VtYhsMz8Ut5Y4GAFbv3Ln8SI0YWj8R0OA6-b#J49pf(smWDm6rusW!5;+%wM= z%$(WCoLO9vr&o;S%(875Rr8Rluy=s_XZ-23^M@NNw668|25hC*IzljW`jyEB7x-=W z`}6)hAj*RleLKrsef^fo_+xet6<>l6thY11SO}WgZB^kNa-rc3bsj9Equl3ES6FJHjJWj<^Iy6safm-+ zn6*jI?{=9G*8FD}KH<>|`EgS`u(wOvlrqiU}hK{a2D50}ylU#MKWt)m}p zZj2dbr>d`j54wY-j;NoAU2<+DTsDhSTEWy?b1=F2ea#26gOWPeTh3(#>Mq+gHMIO< z5lRh?qWtF@pT6)n_|pjMu)4&#zK}xl-bNH%L8BpaSoQ^PM-P7Z>v736$0b_M>A6?A zJLE0Jc(N0QP$}+pDRqZo{&z4Ts4t}l;SEVO{BRa}_=k#c;T`h`O7JA1G?0Omc98zP zoiTJn$CjqCz2T#bjsW-5^gYd$fqQp8*|D^vh_?%dw5ORyzHJ9gA94iT0d7;(5(`bz zMzA_hfML9(E_pz!f&{Qnqc6av?Gk{ldY{L2k*B{SG}&e%WE0KvUjG<4s(U>6rzdk0 zlMbspl;o$x)-rnrzt5m|Z0B_YUf!2O_$B3PARcgrb6I^I-Dz%A(b^+}6DW@UA3d=j z^9uMQ203aE*TNq?R47Yx+xO<^BBn8AGhJdHK@Dcd0Ho=^RC|rb>i*zfK}1Z)#$GfA z2RY-*gfb$QwNFW?<<(mh6e7QZD)$8wfRR}ey$6b}F)627&UX81&6olqyHOgO3Cqh2 zEEqhdIF1ek&ed;+hX(8JRvFXOrp+IGi$EjuE*;l;aHpyuLvOo~f(p>qM3Cw&ogoMn zBstl7z06&1F7PTP(Pku*a$g)4Uyj#R7}?6VcgF#(bePp8Oq$0Q;~pXze3&7}i;BI5o1kJcSSI94)^&-XW6BA9lg%F$YFeAc7ag5}1&hd@keU{=Vp6wSL`? zh2C*6&T_^%Ij$h9I=f9Y2izx;`Ud&$pn=Z)irSAq+iB6ii)+T()ZGSVf>w(bX=V87 zp3io+`Hb&+AmghVKIxFXLMU1n@~T%$(rnv32wffk3o_P44cH_cz)OSos3VqU=YLQ! zyHuC5ST!2iB3iv$n=JA=8!ZZKonc$@2}|vb45f~(9i&z!9*z953!KjUAOErvG!G_J z`qME4DxlzVPo8-W4ka-_)suERc!&)oHD=Xg{ZZGpTD{JA2h0G%FhUw;BD}?VmGAh@ zRF=M>#7Io~b7pIw67O#o+9m#A$o1t8fAVp1Y#ncyGG76x-U=c+f099N4pK(9G{(J4XsbYfU%p(m zmSglDyfsi`5{8Il!Mv_poNW%QO0zuyuLKT4`U}oMn)ZAnC`jAYUO)qACDez&Y414y eA&}DqdrI=Y%+0O2qfA*8ybBK(l5?$N5cMw;M9ZB3 diff --git a/tabpy/tools/__init__.py b/tabpy/tools/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tabpy/tools/client.py b/tabpy/tools/client.py deleted file mode 100644 index f9e90023..00000000 --- a/tabpy/tools/client.py +++ /dev/null @@ -1,389 +0,0 @@ -import copy -from re import compile -import time -import requests - -from .rest import RequestsNetworkWrapper, ServiceClient - -from .rest_client import RESTServiceClient, Endpoint - -from .custom_query_object import CustomQueryObject -import os -import logging - -logger = logging.getLogger(__name__) - -_name_checker = compile(r"^[\w -]+$") - - -def _check_endpoint_type(name): - if not isinstance(name, str): - raise TypeError("Endpoint name must be a string") - - if name == "": - raise ValueError("Endpoint name cannot be empty") - - -def _check_hostname(name): - _check_endpoint_type(name) - hostname_checker = compile(r"^^http(s)?://[\w.-]+(/)?(:\d+)?(/)?$") - - if not hostname_checker.match(name): - raise ValueError( - f"endpoint name {name} should be in http(s)://" - "[:] and hostname may consist only of: " - "a-z, A-Z, 0-9, underscore and hyphens." - ) - - -def _check_endpoint_name(name): - """Checks that the endpoint name is valid by comparing it with an RE and - checking that it is not reserved.""" - _check_endpoint_type(name) - - if not _name_checker.match(name): - raise ValueError( - f"endpoint name {name} can only contain: a-z, A-Z, 0-9," - " underscore, hyphens and spaces." - ) - - -class Client: - def __init__(self, endpoint, query_timeout=1000): - """ - Connects to a running server. - - The class constructor takes a server address which is then used to - connect for all subsequent member APIs. - - Parameters - ---------- - endpoint : str, optional - The server URL. - - query_timeout : float, optional - The timeout for query operations. - """ - _check_hostname(endpoint) - - self._endpoint = endpoint - - session = requests.session() - session.verify = False - requests.packages.urllib3.disable_warnings() - - # Setup the communications layer. - network_wrapper = RequestsNetworkWrapper(session) - service_client = ServiceClient(self._endpoint, network_wrapper) - - self._service = RESTServiceClient(service_client) - if not type(query_timeout) in (int, float) or query_timeout <= 0: - query_timeout = 0.0 - self._service.query_timeout = query_timeout - - def __repr__(self): - return ( - "<" - + self.__class__.__name__ - + " object at " - + hex(id(self)) - + " connected to " - + repr(self._endpoint) - + ">" - ) - - def get_status(self): - """ - Gets the status of the deployed endpoints. - - Returns - ------- - dict - Keys are endpoints and values are dicts describing the state of - the endpoint. - - Examples - -------- - .. sourcecode:: python - { - u'foo': { - u'status': u'LoadFailed', - u'last_error': u'error mesasge', - u'version': 1, - u'type': u'model', - }, - } - """ - return self._service.get_status() - - # - # Query - # - - @property - def query_timeout(self): - """The timeout for queries in milliseconds.""" - return self._service.query_timeout - - @query_timeout.setter - def query_timeout(self, value): - if type(value) in (int, float) and value > 0: - self._service.query_timeout = value - - def query(self, name, *args, **kwargs): - """Query an endpoint. - - Parameters - ---------- - name : str - The name of the endpoint. - - *args : list of anything - Ordered parameters to the endpoint. - - **kwargs : dict of anything - Named parameters to the endpoint. - - Returns - ------- - dict - Keys are: - model: the name of the endpoint - version: the version used. - response: the response to the query. - uuid : a unique id for the request. - """ - return self._service.query(name, *args, **kwargs) - - # - # Endpoints - # - - def get_endpoints(self, type=None): - """Returns all deployed endpoints. - - Examples - -------- - .. sourcecode:: python - {"clustering": - {"description": "", - "docstring": "-- no docstring found in query function --", - "creation_time": 1469511182, - "version": 1, - "dependencies": [], - "last_modified_time": 1469511182, - "type": "model", - "target": null}, - "add": { - "description": "", - "docstring": "-- no docstring found in query function --", - "creation_time": 1469505967, - "version": 1, - "dependencies": [], - "last_modified_time": 1469505967, - "type": "model", - "target": null} - } - """ - return self._service.get_endpoints(type) - - def _get_endpoint_upload_destination(self): - """Returns the endpoint upload destination.""" - return self._service.get_endpoint_upload_destination()["path"] - - def deploy(self, name, obj, description="", schema=None, override=False): - """Deploys a Python function as an endpoint in the server. - - Parameters - ---------- - name : str - A unique identifier for the endpoint. - - obj : function - Refers to a user-defined function with any signature. However both - input and output of the function need to be JSON serializable. - - description : str, optional - The description for the endpoint. This string will be returned by - the ``endpoints`` API. - - schema : dict, optional - The schema of the function, containing information about input and - output parameters, and respective examples. Providing a schema for - a deployed function lets other users of the service discover how to - use it. Refer to schema.generate_schema for more information on - how to generate the schema. - - override : bool - Whether to override (update) an existing endpoint. If False and - there is already an endpoint with that name, it will raise a - RuntimeError. If True and there is already an endpoint with that - name, it will deploy a new version on top of it. - - See Also - -------- - remove, get_endpoints - """ - endpoint = self.get_endpoints().get(name) - version = 1 - if endpoint: - if not override: - raise RuntimeError( - f"An endpoint with that name ({name}) already" - ' exists. Use "override = True" to force update ' - "an existing endpoint." - ) - - version = endpoint.version + 1 - - obj = self._gen_endpoint(name, obj, description, version, schema) - - self._upload_endpoint(obj) - - if version == 1: - self._service.add_endpoint(Endpoint(**obj)) - else: - self._service.set_endpoint(Endpoint(**obj)) - - self._wait_for_endpoint_deployment(obj["name"], obj["version"]) - - def remove(self, name): - '''Removes an endpoint dict. - - Parameters - ---------- - name : str - Endpoint name to remove''' - self._service.remove_endpoint(name) - - def _gen_endpoint(self, name, obj, description, version=1, schema=None): - """Generates an endpoint dict. - - Parameters - ---------- - name : str - Endpoint name to add or update - - obj : func - Object that backs the endpoint. See add() for a complete - description. - - description : str - Description of the endpoint - - version : int - The version. Defaults to 1. - - Returns - ------- - dict - Keys: - name : str - The name provided. - - version : int - The version provided. - - description : str - The provided description. - - type : str - The type of the endpoint. - - endpoint_obj : object - The wrapper around the obj provided that can be used to - generate the code and dependencies for the endpoint. - - Raises - ------ - TypeError - When obj is not one of the expected types. - """ - # check for invalid PO names - _check_endpoint_name(name) - - if description is None: - description = obj.__doc__.strip() or "" if isinstance(obj.__doc__, str) else "" - - endpoint_object = CustomQueryObject(query=obj, description=description,) - - return { - "name": name, - "version": version, - "description": description, - "type": "model", - "endpoint_obj": endpoint_object, - "dependencies": endpoint_object.get_dependencies(), - "methods": endpoint_object.get_methods(), - "required_files": [], - "required_packages": [], - "schema": copy.copy(schema), - } - - def _upload_endpoint(self, obj): - """Sends the endpoint across the wire.""" - endpoint_obj = obj["endpoint_obj"] - - dest_path = self._get_endpoint_upload_destination() - - # Upload the endpoint - obj["src_path"] = os.path.join( - dest_path, "endpoints", obj["name"], str(obj["version"]) - ) - - endpoint_obj.save(obj["src_path"]) - - def _wait_for_endpoint_deployment( - self, endpoint_name, version=1, interval=1.0, - ): - """ - Waits for the endpoint to be deployed by calling get_status() and - checking the versions deployed of the endpoint against the expected - version. If all the versions are equal to or greater than the version - expected, then it will return. Uses time.sleep(). - """ - logger.info( - f"Waiting for endpoint {endpoint_name} to deploy to " f"version {version}" - ) - start = time.time() - while True: - ep_status = self.get_status() - try: - ep = ep_status[endpoint_name] - except KeyError: - logger.info( - f"Endpoint {endpoint_name} doesn't " "exist in endpoints yet" - ) - else: - logger.info(f"ep={ep}") - - if ep["status"] == "LoadFailed": - raise RuntimeError(f'LoadFailed: {ep["last_error"]}') - - elif ep["status"] == "LoadSuccessful": - if ep["version"] >= version: - logger.info("LoadSuccessful") - break - else: - logger.info("LoadSuccessful but wrong version") - - if time.time() - start > 10: - raise RuntimeError("Waited more then 10s for deployment") - - logger.info(f"Sleeping {interval}...") - time.sleep(interval) - - def set_credentials(self, username, password): - """ - Set credentials for all the TabPy client-server communication - where client is tabpy-tools and server is tabpy-server. - - Parameters - ---------- - username : str - User name (login). Username is case insensitive. - - password : str - Password in plain text. - """ - self._service.set_credentials(username, password) diff --git a/tabpy/tools/custom_query_object.py b/tabpy/tools/custom_query_object.py deleted file mode 100644 index 18a149b8..00000000 --- a/tabpy/tools/custom_query_object.py +++ /dev/null @@ -1,83 +0,0 @@ -import logging -from .query_object import QueryObject as _QueryObject - - -logger = logging.getLogger(__name__) - - -class CustomQueryObject(_QueryObject): - def __init__(self, query, description=""): - """Create a new CustomQueryObject. - - Parameters - ----------- - - query : function - Function that defines a custom query method. The query can have any - signature, but input and output of the query needs to be JSON - serializable. - - description : str - The description of the custom query object - - """ - super().__init__(description) - - self.custom_query = query - - def query(self, *args, **kwargs): - """Query the custom defined query method using the given input. - - Parameters - ---------- - args : list - positional arguments to the query - - kwargs : dict - keyword arguments to the query - - Returns - ------- - out: object. - The results depends on the implementation of the query method. - Typically the return value will be whatever that function returns. - - See Also - -------- - QueryObject - """ - # include the dependent files in sys path so that the query can run - # correctly - - try: - logger.debug( - "Running custom query with arguments " f"({args}, {kwargs})..." - ) - ret = self.custom_query(*args, **kwargs) - except Exception as e: - logger.exception( - "Exception hit when running custom query, error: " f"{str(e)}" - ) - raise - - logger.debug(f"Received response {ret}") - try: - return self._make_serializable(ret) - except Exception as e: - logger.exception( - "Cannot properly serialize custom query result, " f"error: {str(e)}" - ) - raise - - def get_doc_string(self): - """Get doc string from customized query""" - if self.custom_query.__doc__ is not None: - return self.custom_query.__doc__ - else: - return "-- no docstring found in query function --" - - def get_methods(self): - return [self.get_query_method()] - - def get_query_method(self): - return {"method": "query"} diff --git a/tabpy/tools/query_object.py b/tabpy/tools/query_object.py deleted file mode 100644 index 5ccbc109..00000000 --- a/tabpy/tools/query_object.py +++ /dev/null @@ -1,108 +0,0 @@ -import abc -import logging -import os -import json -import shutil - -import cloudpickle as _cloudpickle - - -logger = logging.getLogger(__name__) - - -class QueryObject(abc.ABC): - """ - Derived class needs to implement the following interface: - * query() -- given input, return query result - * get_doc_string() -- returns documentation for the Query Object - """ - - def __init__(self, description=""): - self.description = description - - def get_dependencies(self): - """All endpoints this endpoint depends on""" - return [] - - @abc.abstractmethod - def query(self, input): - """execute query on the provided input""" - pass - - @abc.abstractmethod - def get_doc_string(self): - """Returns documentation for the query object - - By default, this method returns the docstring for 'query' method - Derived class may overwrite this method to dynamically create docstring - """ - pass - - def save(self, path): - """ Save query object to the given local path - - Parameters - ---------- - path : str - The location to save the query object to - """ - if os.path.exists(path): - logger.warning( - f'Overwriting existing file "{path}" when saving query object' - ) - rm_fn = os.remove if os.path.isfile(path) else shutil.rmtree - rm_fn(path) - self._save_local(path) - - def _save_local(self, path): - """Save current query object to local path - """ - try: - os.makedirs(path) - except OSError as e: - import errno - - if e.errno == errno.EEXIST and os.path.isdir(path): - pass - else: - raise - - with open(os.path.join(path, "pickle_archive"), "wb") as f: - _cloudpickle.dump(self, f) - - @classmethod - def load(cls, path): - """ Load query object from given path - """ - new_po = None - new_po = cls._load_local(path) - - logger.info(f'Loaded query object "{type(new_po).__name__}" successfully') - - return new_po - - @classmethod - def _load_local(cls, path): - path = os.path.abspath(os.path.expanduser(path)) - with open(os.path.join(path, "pickle_archive"), "rb") as f: - return _cloudpickle.load(f) - - @classmethod - def _make_serializable(cls, result): - """Convert a result from object query to python data structure that can - easily serialize over network - """ - try: - json.dumps(result) - except TypeError: - raise TypeError( - "Result from object query is not json serializable: " f"{result}" - ) - - return result - - # Returns an array of dictionary that contains the methods and their - # corresponding schema information. - @abc.abstractmethod - def get_methods(self): - return None diff --git a/tabpy/tools/rest.py b/tabpy/tools/rest.py deleted file mode 100644 index f200c250..00000000 --- a/tabpy/tools/rest.py +++ /dev/null @@ -1,423 +0,0 @@ -import abc -from collections.abc import MutableMapping -import logging -import requests -from requests.auth import HTTPBasicAuth -from re import compile -import json as json - - -logger = logging.getLogger(__name__) - - -class ResponseError(Exception): - """Raised when we get an unexpected response.""" - - def __init__(self, response): - super().__init__("Unexpected server response") - self.response = response - self.status_code = response.status_code - - try: - r = response.json() - self.info = r["info"] - self.message = response.json()["message"] - except (json.JSONDecodeError, KeyError): - self.info = None - self.message = response.text - - def __str__(self): - return f"({self.status_code}) " f"{self.message} " f"{self.info}" - - -class RequestsNetworkWrapper: - """The NetworkWrapper wraps the underlying network connection to simplify - the interface a bit. This can be replaced with something that can be built - on some other type of network connection, such as PyCURL. - - This version requires you to instantiate a requests session object to your - liking. It will create a generic session for you if you don't specify it, - which you can modify later. - - For authentication, use:: - - session.auth = (username, password) - """ - - def __init__(self, session=None): - # Set .auth as appropriate. - if session is None: - session = requests.session() - - self.session = session - self.auth = None - - @staticmethod - def raise_error(response): - logger.error( - f"Error with server response. code={response.status_code}; " - f"text={response.text}" - ) - - raise ResponseError(response) - - @staticmethod - def _remove_nones(data): - if isinstance(data, dict): - for k in [k for k, v in data.items() if v is None]: - del data[k] - - def _encode_request(self, data): - self._remove_nones(data) - - if data is not None: - return json.dumps(data) - else: - return None - - def GET(self, url, data, timeout=None): - """Issues a GET request to the URL with the data specified. Returns an - object that is parsed from the response JSON.""" - self._remove_nones(data) - - logger.info(f"GET {url} with {data}") - - response = self.session.get(url, params=data, timeout=timeout, auth=self.auth) - if response.status_code != 200: - self.raise_error(response) - logger.info(f"response={response.text}") - - if response.text == "": - return dict() - else: - return response.json() - - def POST(self, url, data, timeout=None): - """Issues a POST request to the URL with the data specified. Returns an - object that is parsed from the response JSON.""" - data = self._encode_request(data) - - logger.info(f"POST {url} with {data}") - response = self.session.post( - url, - data=data, - headers={"content-type": "application/json"}, - timeout=timeout, - auth=self.auth, - ) - - if response.status_code not in (200, 201): - self.raise_error(response) - - return response.json() - - def PUT(self, url, data, timeout=None): - """Issues a PUT request to the URL with the data specified. Returns an - object that is parsed from the response JSON.""" - data = self._encode_request(data) - - logger.info(f"PUT {url} with {data}") - - response = self.session.put( - url, - data=data, - headers={"content-type": "application/json"}, - timeout=timeout, - auth=self.auth, - ) - if response.status_code != 200: - self.raise_error(response) - - return response.json() - - def DELETE(self, url, data, timeout=None): - """ - Issues a DELETE request to the URL with the data specified. Returns an - object that is parsed from the response JSON. - """ - if data is not None: - data = json.dumps(data) - - logger.info(f"DELETE {url} with {data}") - - response = self.session.delete(url, data=data, timeout=timeout, auth=self.auth) - - if response.status_code <= 499 and response.status_code >= 400: - raise RuntimeError(response.text) - - if response.status_code not in (200, 201, 204): - raise RuntimeError( - f"Error with server response code: {response.status_code}" - ) - - def set_credentials(self, username, password): - """ - Set credentials for all the TabPy client-server communication - where client is tabpy-tools and server is tabpy-server. - - Parameters - ---------- - username : str - User name (login). Username is case insensitive. - - password : str - Password in plain text. - """ - logger.info(f"Setting credentials (username: {username})") - self.auth = HTTPBasicAuth(username, password) - - -class ServiceClient: - """ - A generic service client. - - This will take an endpoint URL and a network_wrapper. You can use the - RequestsNetworkWrapper if you want to use the requests module. The - endpoint URL is prepended to all the requests and forwarded to the network - wrapper. - """ - - def __init__(self, endpoint, network_wrapper=None): - if network_wrapper is None: - network_wrapper = RequestsNetworkWrapper(session=requests.session()) - - self.network_wrapper = network_wrapper - - pattern = compile(".*(:[0-9]+)$") - if not endpoint.endswith("/") and not pattern.match(endpoint): - logger.warning(f"endpoint {endpoint} does not end with '/': appending.") - endpoint = endpoint + "/" - - self.endpoint = endpoint - - def GET(self, url, data=None, timeout=None): - """Prepends self.endpoint to the url and issues a GET request.""" - return self.network_wrapper.GET(self.endpoint + url, data, timeout) - - def POST(self, url, data=None, timeout=None): - """Prepends self.endpoint to the url and issues a POST request.""" - return self.network_wrapper.POST(self.endpoint + url, data, timeout) - - def PUT(self, url, data=None, timeout=None): - """Prepends self.endpoint to the url and issues a PUT request.""" - return self.network_wrapper.PUT(self.endpoint + url, data, timeout) - - def DELETE(self, url, data=None, timeout=None): - """Prepends self.endpoint to the url and issues a DELETE request.""" - self.network_wrapper.DELETE(self.endpoint + url, data, timeout) - - def set_credentials(self, username, password): - """ - Set credentials for all the TabPy client-server communication - where client is tabpy-tools and server is tabpy-server. - - Parameters - ---------- - username : str - User name (login). Username is case insensitive. - - password : str - Password in plain text. - """ - self.network_wrapper.set_credentials(username, password) - - -class RESTProperty: - """A descriptor that will control the type of value stored.""" - - def __init__(self, type, from_json=lambda x: x, to_json=lambda x: x, doc=None): - self.__doc__ = doc - self.type = type - self.from_json = from_json - self.to_json = to_json - - def __get__(self, instance, _): - if instance: - try: - return getattr(instance, self.name) - except AttributeError: - raise AttributeError(f"{self.name} has not been set yet.") - else: - return self - - def __set__(self, instance, value): - if value is not None and not isinstance(value, self.type): - value = self.type(value) - - setattr(instance, self.name, value) - - def __delete__(self, instance): - delattr(instance, self.name) - - -class _RESTMetaclass(abc.ABCMeta): - """The metaclass for RESTObjects. - - This will look into the attributes for the class. If they are a - RESTProperty, then it will add it to the __rest__ set and give it its - name. - - If the bases have __rest__, then it will add them to the __rest__ set as - well. - """ - - def __init__(self, name, bases, dict): - super().__init__(name, bases, dict) - - self.__rest__ = set() - for base in bases: - self.__rest__.update(getattr(base, "__rest__", set())) - - for k, v in dict.items(): - if isinstance(v, RESTProperty): - v.__dict__["name"] = "_" + k - self.__rest__.add(k) - - -class RESTObject(MutableMapping, metaclass=_RESTMetaclass): - """A base class that has methods generally useful for interacting with - REST objects. The attributes are accessible either as dict keys or as - attributes. The object also behaves like a dict, even replicating the - repr() functionality. - - Attributes - ---------- - - __rest__ : set of str - A set of all the rest attribute names. This is generated automatically - and should include all of the base classes' __rest__ as well as any - addition RESTProperty. - - """ - - """ __metaclass__ = _RESTMetaclass""" - - def __init__(self, **kwargs): - """Creates a new instance of the RESTObject. - - Parameters - ---------- - - The parameters depend on __rest__. Each item in __rest__ is searched - for. If found, it is assigned to the instance. Additional parameters - are ignored. - - """ - logger.info(f"Initializing {self.__class__.__name__} from {kwargs}") - for attr in self.__rest__: - if attr in kwargs: - setattr(self, attr, kwargs.pop(attr)) - - def __repr__(self): - return ( - "{" + ", ".join([repr(k) + ": " + repr(v) for k, v in self.items()]) + "}" - ) - - @classmethod - def from_json(cls, data): - """Returns a new class object with data populated from json.loads().""" - attrs = {} - for attr in cls.__rest__: - try: - value = data[attr] - except KeyError: - pass - else: - prop = cls.__dict__[attr] - attrs[attr] = prop.from_json(value) - return cls(**attrs) - - def to_json(self): - """Returns a dict representing this object. This dict will be sent to - json.dumps(). - - The keys are the items in __rest__ and the values are the current - values. If missing, it is not included. - """ - result = {} - for attr in self.__rest__: - prop = getattr(self.__class__, attr) - try: - result[attr] = prop.to_json(getattr(self, attr)) - except AttributeError: - pass - - return result - - def __eq__(self, other): - return isinstance(self, type(other)) and all( - (getattr(self, a) == getattr(other, a) for a in self.__rest__) - ) - - def __len__(self): - return len([a for a in self.__rest__ if hasattr(self, "_" + a)]) - - def __iter__(self): - return iter([a for a in self.__rest__ if hasattr(self, "_" + a)]) - - def __getitem__(self, item): - if item not in self.__rest__: - raise KeyError(item) - try: - return getattr(self, item) - except AttributeError: - raise KeyError(item) - - def __setitem__(self, item, value): - if item not in self.__rest__: - raise KeyError(item) - setattr(self, item, value) - - def __delitem__(self, item): - if item not in self.__rest__: - raise KeyError(item) - try: - delattr(self, "_" + item) - except AttributeError: - raise KeyError(item) - - def __contains__(self, item): - return item in self.__rest__ - - -def enum(*values, **kwargs): - """Generates an enum function that only accepts particular values. Other - values will raise a ValueError. - - Parameters - ---------- - - values : list - These are the acceptable values. - - type : type - The acceptable types of values. Values will be converted before being - checked against the allowed values. If not specified, no conversion - will be performed. - - Example - ------- - - >>> my_enum = enum(1, 2, 3, 4, 5, type=int) - >>> a = my_enum(1) - >>> b = my_enum(2) - >>> c = mu_enum(6) # Raises ValueError - - """ - if len(values) < 1: - raise ValueError("At least one value is required.") - enum_type = kwargs.pop("type", str) - if kwargs: - raise TypeError(f'Unexpected parameters: {", ".join(kwargs.keys())}') - - def __new__(cls, value): - if value not in cls.values: - raise ValueError( - f"{value} is an unexpected value. " f"Expected one of {cls.values}" - ) - - return super(enum, cls).__new__(cls, value) - - enum = type("Enum", (enum_type,), {"values": values, "__new__": __new__}) - - return enum diff --git a/tabpy/tools/rest_client.py b/tabpy/tools/rest_client.py deleted file mode 100644 index eb0ef211..00000000 --- a/tabpy/tools/rest_client.py +++ /dev/null @@ -1,252 +0,0 @@ -from .rest import RESTObject, RESTProperty -from datetime import datetime - - -def from_epoch(value): - if isinstance(value, datetime): - return value - else: - return datetime.utcfromtimestamp(value) - - -def to_epoch(value): - return (value - datetime(1970, 1, 1)).total_seconds() - - -class Endpoint(RESTObject): - """Represents an endpoint. - - Note that not every attribute is returned as part of the GET. - - Attributes - ---------- - - name : str - The name of the endpoint. Valid names include ``[a-zA-Z0-9_\\- ]+`` - type : str - The type of endpoint. The types include "alias", "model". - version : int - The version of this endpoint. Initial versions have version on 1. New - versions increment this by 1. - description : str - A human-readable description of the endpoint. - dependencies: list - A list of endpoints that this endpoint depends on. - methods : list - ??? - """ - - name = RESTProperty(str) - type = RESTProperty(str) - version = RESTProperty(int) - description = RESTProperty(str) - dependencies = RESTProperty(list) - methods = RESTProperty(list) - creation_time = RESTProperty(datetime, from_epoch, to_epoch) - last_modified_time = RESTProperty(datetime, from_epoch, to_epoch) - evaluator = RESTProperty(str) - schema_version = RESTProperty(int) - schema = RESTProperty(str) - - def __new__(cls, **kwargs): - """Dispatch to the appropriate class.""" - cls = {"alias": AliasEndpoint, "model": ModelEndpoint}[kwargs["type"]] - - """return object.__new__(cls, **kwargs)""" - """ modified for Python 3""" - return object.__new__(cls) - - def __eq__(self, other): - return ( - self.name == other.name - and self.type == other.type - and self.version == other.version - and self.description == other.description - and self.dependencies == other.dependencies - and self.methods == other.methods - and self.evaluator == other.evaluator - and self.schema_version == other.schema_version - and self.schema == other.schema - ) - - -class ModelEndpoint(Endpoint): - """Represents a model endpoint. - - src_path : str - - The local file path to the source of this object. - - required_files : str - - The local file path to the directory containing the - required files. - - required_packages : str - - The local file path to the directory containing the - required packages. - - """ - - src_path = RESTProperty(str) - required_files = RESTProperty(list) - required_packages = RESTProperty(list) - required_packages_dst_path = RESTProperty(str) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.type = "model" - - def __eq__(self, other): - return ( - super().__eq__(other) - and self.required_files == other.required_files - and self.required_packages == other.required_packages - ) - - -class AliasEndpoint(Endpoint): - """Represents an alias Endpoint. - - target : str - - The endpoint that this is an alias for. - - """ - - target = RESTProperty(str) - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.type = "alias" - - -class RESTServiceClient: - """A thin client for the REST Service.""" - - def __init__(self, service_client): - self.service_client = service_client - self.query_timeout = None - - def get_info(self): - """Returns the /info""" - return self.service_client.GET("info") - - def query(self, name, *args, **kwargs): - """Performs a query. Either specify *args or **kwargs, not both. - Respects query_timeout.""" - if args and kwargs: - raise ValueError( - "Mixing of keyword arguments and positional arguments when " - "querying an endpoint is not supported." - ) - return self.service_client.POST( - "query/" + name, data={"data": args or kwargs}, timeout=self.query_timeout - ) - - def get_endpoint_upload_destination(self): - """Returns a dict representing where endpoint data should be uploaded. - - Returns - ------- - dict - Keys include: - * path: a local file path. - - Note: In the future, other paths and parameters may be supported. - - Note: At this time, the response should not change over time. - """ - return self.service_client.GET("configurations/endpoint_upload_destination") - - def get_endpoints(self, type=None): - """Returns endpoints from the management API. - - Parameters - ---------- - - type : str - The type of endpoint to return. None will include all endpoints. - Other options are 'model' and 'alias'. - """ - result = {} - for name, attrs in self.service_client.GET("endpoints", {"type": type}).items(): - endpoint = Endpoint.from_json(attrs) - endpoint.name = name - result[name] = endpoint - return result - - def get_endpoint(self, endpoint_name): - """Returns an endpoints from the management API given its name. - - Parameters - ---------- - - endpoint_name : str - - The name of the endpoint. - """ - ((name, attrs),) = self.service_client.GET("endpoints/" + endpoint_name).items() - endpoint = Endpoint.from_json(attrs) - endpoint.name = name - return endpoint - - def add_endpoint(self, endpoint): - """Adds an endpoint through the management API. - - Parameters - ---------- - - endpoint : Endpoint - """ - return self.service_client.POST("endpoints", endpoint.to_json()) - - def set_endpoint(self, endpoint): - """Updates an endpoint through the management API. - - Parameters - ---------- - - endpoint : Endpoint - - The endpoint to update. - """ - return self.service_client.PUT("endpoints/" + endpoint.name, endpoint.to_json()) - - def remove_endpoint(self, endpoint_name): - """Deletes an endpoint through the management API. - - Parameters - ---------- - - endpoint_name : str - - The endpoint to delete. - """ - self.service_client.DELETE("endpoints/" + endpoint_name) - - def get_status(self): - """Returns the status of the server. - - Returns - ------- - - dict - """ - return self.service_client.GET("status") - - def set_credentials(self, username, password): - """ - Set credentials for all the TabPy client-server communication - where client is tabpy-tools and server is tabpy-server. - - Parameters - ---------- - username : str - User name (login). Username is case insensitive. - - password : str - Password in plain text. - """ - self.service_client.set_credentials(username, password) diff --git a/tabpy/tools/schema.py b/tabpy/tools/schema.py deleted file mode 100644 index 27b2e425..00000000 --- a/tabpy/tools/schema.py +++ /dev/null @@ -1,108 +0,0 @@ -import logging -import genson -import jsonschema - - -logger = logging.getLogger(__name__) - - -def _generate_schema_from_example_and_description(input, description): - """ - With an example input, a schema is automatically generated that conforms - to the example in json-schema.org. The description given by the users - is then added to the schema. - """ - s = genson.SchemaBuilder(None) - s.add_object(input) - input_schema = s.to_schema() - - if description is not None: - if "properties" in input_schema: - # Case for input = {'x':1}, input_description='not a dict' - if not isinstance(description, dict): - msg = f"{input} and {description} do not match" - logger.error(msg) - raise Exception(msg) - - for key in description: - # Case for input = {'x':1}, - # input_description={'x':'x value', 'y':'y value'} - if key not in input_schema["properties"]: - msg = f"{key} not found in {input}" - logger.error(msg) - raise Exception(msg) - else: - input_schema["properties"][key]["description"] = description[key] - else: - if isinstance(description, dict): - raise Exception(f"{input} and {description} do not match") - else: - input_schema["description"] = description - - try: - # This should not fail unless there are bugs with either genson or - # jsonschema. - jsonschema.validate(input, input_schema) - except Exception as e: - logger.error(f"Internal error validating schema: {str(e)}") - raise - - return input_schema - - -def generate_schema(input, output, input_description=None, output_description=None): - """ - Generate schema from a given sample input and output. - A generated schema can be passed to a server together with a function to - annotate it with information about input and output parameters, and - examples thereof. The schema needs to follow the conventions of JSON Schema - (see json-schema.org). - - Parameters - ----------- - input : any python type | dict - output: any python type | dict - input_description : str | dict, optional - output_description : str | dict, optional - - References - ----------- - - `Json Schema ` - - Examples - ---------- - .. sourcecode:: python - For just one input parameter, state the example directly. - >>> from tabpy.tools.schema import generate_schema - >>> schema = generate_schema( - input=5, - output=25, - input_description='input value', - output_description='the squared value of input') - >>> schema - {'sample': 5, - 'input': {'type': 'integer', 'description': 'input value'}, - 'output': {'type': 'integer', 'description': 'the squared value of input'}} - For two or more input parameters, specify them using a dictionary. - >>> import graphlab - >>> schema = generate_schema( - input={'x': 3, 'y': 2}, - output=6, - input_description={'x': 'value of x', - 'y': 'value of y'}, - output_description='x times y') - >>> schema - {'sample': {'y': 2, 'x': 3}, - 'input': {'required': ['x', 'y'], - 'type': 'object', - 'properties': {'y': {'type': 'integer', 'description': 'value of y'}, - 'x': {'type': 'integer', 'description': 'value of x'}}}, - 'output': {'type': 'integer', 'description': 'x times y'}} - """ # noqa: E501 - input_schema = _generate_schema_from_example_and_description( - input, input_description - ) - output_schema = _generate_schema_from_example_and_description( - output, output_description - ) - return {"input": input_schema, "sample": input, "output": output_schema} From bf613d209885025bc62e91a712551f90351cf162 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Mon, 27 Jul 2020 16:05:28 -0700 Subject: [PATCH 44/61] Restore tabpy_tools and tabpy_server names --- tabpy/tabpy_server/__init__.py | 0 tabpy/tabpy_server/app/ConfigParameters.py | 17 + tabpy/tabpy_server/app/SettingsParameters.py | 17 + tabpy/tabpy_server/app/__init__.py | 0 tabpy/tabpy_server/app/app.py | 430 ++++++++++++ tabpy/tabpy_server/app/util.py | 88 +++ tabpy/tabpy_server/common/__init__.py | 0 tabpy/tabpy_server/common/default.conf | 62 ++ .../tabpy_server/common/endpoint_file_mgr.py | 94 +++ tabpy/tabpy_server/common/messages.py | 172 +++++ tabpy/tabpy_server/common/util.py | 3 + tabpy/tabpy_server/handlers/__init__.py | 13 + tabpy/tabpy_server/handlers/base_handler.py | 425 ++++++++++++ .../tabpy_server/handlers/endpoint_handler.py | 141 ++++ .../handlers/endpoints_handler.py | 75 ++ .../handlers/evaluation_plane_handler.py | 142 ++++ tabpy/tabpy_server/handlers/main_handler.py | 7 + .../handlers/management_handler.py | 150 ++++ .../handlers/query_plane_handler.py | 233 +++++++ .../handlers/service_info_handler.py | 23 + tabpy/tabpy_server/handlers/status_handler.py | 29 + .../handlers/upload_destination_handler.py | 20 + tabpy/tabpy_server/handlers/util.py | 32 + tabpy/tabpy_server/management/__init__.py | 0 tabpy/tabpy_server/management/state.py | 643 ++++++++++++++++++ tabpy/tabpy_server/management/util.py | 46 ++ tabpy/tabpy_server/psws/__init__.py | 0 tabpy/tabpy_server/psws/callbacks.py | 205 ++++++ tabpy/tabpy_server/psws/python_service.py | 275 ++++++++ tabpy/tabpy_server/state.ini.template | 15 + tabpy/tabpy_server/static/index.html | 72 ++ tabpy/tabpy_server/static/tableau.png | Bin 0 -> 33575 bytes tabpy/tabpy_tools/__init__.py | 0 tabpy/tabpy_tools/client.py | 389 +++++++++++ tabpy/tabpy_tools/custom_query_object.py | 83 +++ tabpy/tabpy_tools/query_object.py | 108 +++ tabpy/tabpy_tools/rest.py | 423 ++++++++++++ tabpy/tabpy_tools/rest_client.py | 252 +++++++ tabpy/tabpy_tools/schema.py | 108 +++ 39 files changed, 4792 insertions(+) create mode 100644 tabpy/tabpy_server/__init__.py create mode 100644 tabpy/tabpy_server/app/ConfigParameters.py create mode 100644 tabpy/tabpy_server/app/SettingsParameters.py create mode 100644 tabpy/tabpy_server/app/__init__.py create mode 100644 tabpy/tabpy_server/app/app.py create mode 100644 tabpy/tabpy_server/app/util.py create mode 100644 tabpy/tabpy_server/common/__init__.py create mode 100644 tabpy/tabpy_server/common/default.conf create mode 100644 tabpy/tabpy_server/common/endpoint_file_mgr.py create mode 100644 tabpy/tabpy_server/common/messages.py create mode 100644 tabpy/tabpy_server/common/util.py create mode 100644 tabpy/tabpy_server/handlers/__init__.py create mode 100644 tabpy/tabpy_server/handlers/base_handler.py create mode 100644 tabpy/tabpy_server/handlers/endpoint_handler.py create mode 100644 tabpy/tabpy_server/handlers/endpoints_handler.py create mode 100644 tabpy/tabpy_server/handlers/evaluation_plane_handler.py create mode 100644 tabpy/tabpy_server/handlers/main_handler.py create mode 100644 tabpy/tabpy_server/handlers/management_handler.py create mode 100644 tabpy/tabpy_server/handlers/query_plane_handler.py create mode 100644 tabpy/tabpy_server/handlers/service_info_handler.py create mode 100644 tabpy/tabpy_server/handlers/status_handler.py create mode 100644 tabpy/tabpy_server/handlers/upload_destination_handler.py create mode 100644 tabpy/tabpy_server/handlers/util.py create mode 100644 tabpy/tabpy_server/management/__init__.py create mode 100644 tabpy/tabpy_server/management/state.py create mode 100644 tabpy/tabpy_server/management/util.py create mode 100644 tabpy/tabpy_server/psws/__init__.py create mode 100644 tabpy/tabpy_server/psws/callbacks.py create mode 100644 tabpy/tabpy_server/psws/python_service.py create mode 100644 tabpy/tabpy_server/state.ini.template create mode 100644 tabpy/tabpy_server/static/index.html create mode 100644 tabpy/tabpy_server/static/tableau.png create mode 100644 tabpy/tabpy_tools/__init__.py create mode 100644 tabpy/tabpy_tools/client.py create mode 100644 tabpy/tabpy_tools/custom_query_object.py create mode 100644 tabpy/tabpy_tools/query_object.py create mode 100644 tabpy/tabpy_tools/rest.py create mode 100644 tabpy/tabpy_tools/rest_client.py create mode 100644 tabpy/tabpy_tools/schema.py diff --git a/tabpy/tabpy_server/__init__.py b/tabpy/tabpy_server/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy/tabpy_server/app/ConfigParameters.py b/tabpy/tabpy_server/app/ConfigParameters.py new file mode 100644 index 00000000..3feca9b7 --- /dev/null +++ b/tabpy/tabpy_server/app/ConfigParameters.py @@ -0,0 +1,17 @@ +class ConfigParameters: + """ + Configuration settings names + """ + + TABPY_PWD_FILE = "TABPY_PWD_FILE" + TABPY_PORT = "TABPY_PORT" + TABPY_QUERY_OBJECT_PATH = "TABPY_QUERY_OBJECT_PATH" + TABPY_STATE_PATH = "TABPY_STATE_PATH" + TABPY_TRANSFER_PROTOCOL = "TABPY_TRANSFER_PROTOCOL" + TABPY_CERTIFICATE_FILE = "TABPY_CERTIFICATE_FILE" + TABPY_KEY_FILE = "TABPY_KEY_FILE" + TABPY_PWD_FILE = "TABPY_PWD_FILE" + TABPY_LOG_DETAILS = "TABPY_LOG_DETAILS" + TABPY_STATIC_PATH = "TABPY_STATIC_PATH" + TABPY_MAX_REQUEST_SIZE_MB = "TABPY_MAX_REQUEST_SIZE_MB" + TABPY_EVALUATE_TIMEOUT = "TABPY_EVALUATE_TIMEOUT" diff --git a/tabpy/tabpy_server/app/SettingsParameters.py b/tabpy/tabpy_server/app/SettingsParameters.py new file mode 100644 index 00000000..45fb128a --- /dev/null +++ b/tabpy/tabpy_server/app/SettingsParameters.py @@ -0,0 +1,17 @@ +class SettingsParameters: + """ + Application (TabPyApp) settings names + """ + + TransferProtocol = "transfer_protocol" + Port = "port" + ServerVersion = "server_version" + UploadDir = "upload_dir" + CertificateFile = "certificate_file" + KeyFile = "key_file" + StateFilePath = "state_file_path" + ApiVersions = "versions" + LogRequestContext = "log_request_context" + StaticPath = "static_path" + MaxRequestSizeInMb = "max_request_size_in_mb" + EvaluateTimeout = "evaluate_timeout" diff --git a/tabpy/tabpy_server/app/__init__.py b/tabpy/tabpy_server/app/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py new file mode 100644 index 00000000..735218ca --- /dev/null +++ b/tabpy/tabpy_server/app/app.py @@ -0,0 +1,430 @@ +import concurrent.futures +import configparser +import logging +import multiprocessing +import os +import shutil +import signal +import sys +import tabpy +from tabpy.tabpy import __version__ +from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.app.util import parse_pwd_file +from tabpy.tabpy_server.management.state import TabPyState +from tabpy.tabpy_server.management.util import _get_state_from_file +from tabpy.tabpy_server.psws.callbacks import init_model_evaluator, init_ps_server +from tabpy.tabpy_server.psws.python_service import PythonService, PythonServiceHandler +from tabpy.tabpy_server.handlers import ( + EndpointHandler, + EndpointsHandler, + EvaluationPlaneHandler, + QueryPlaneHandler, + ServiceInfoHandler, + StatusHandler, + UploadDestinationHandler, +) +import tornado + + +logger = logging.getLogger(__name__) + + +def _init_asyncio_patch(): + """ + Select compatible event loop for Tornado 5+. + As of Python 3.8, the default event loop on Windows is `proactor`, + however Tornado requires the old default "selector" event loop. + As Tornado has decided to leave this to users to set, MkDocs needs + to set it. See https://github.com/tornadoweb/tornado/issues/2608. + """ + if sys.platform.startswith("win") and sys.version_info >= (3, 8): + import asyncio + try: + from asyncio import WindowsSelectorEventLoopPolicy + except ImportError: + pass # Can't assign a policy which doesn't exist. + else: + if not isinstance(asyncio.get_event_loop_policy(), WindowsSelectorEventLoopPolicy): + asyncio.set_event_loop_policy(WindowsSelectorEventLoopPolicy()) + + +class TabPyApp: + """ + TabPy application class for keeping context like settings, state, etc. + """ + + settings = {} + subdirectory = "" + tabpy_state = None + python_service = None + credentials = {} + + def __init__(self, config_file=None): + if config_file is None: + config_file = os.path.join( + os.path.dirname(__file__), os.path.pardir, "common", "default.conf" + ) + + if os.path.isfile(config_file): + try: + from logging import config + config.fileConfig(config_file, disable_existing_loggers=False) + except KeyError: + logging.basicConfig(level=logging.DEBUG) + + self._parse_config(config_file) + + def run(self): + application = self._create_tornado_web_app() + max_request_size = ( + int(self.settings[SettingsParameters.MaxRequestSizeInMb]) * 1024 * 1024 + ) + logger.info(f"Setting max request size to {max_request_size} bytes") + + init_model_evaluator(self.settings, self.tabpy_state, self.python_service) + + protocol = self.settings[SettingsParameters.TransferProtocol] + ssl_options = None + if protocol == "https": + ssl_options = { + "certfile": self.settings[SettingsParameters.CertificateFile], + "keyfile": self.settings[SettingsParameters.KeyFile], + } + elif protocol != "http": + msg = f"Unsupported transfer protocol {protocol}." + logger.critical(msg) + raise RuntimeError(msg) + + application.listen( + self.settings[SettingsParameters.Port], + ssl_options=ssl_options, + max_buffer_size=max_request_size, + max_body_size=max_request_size, + ) + + logger.info( + "Web service listening on port " + f"{str(self.settings[SettingsParameters.Port])}" + ) + tornado.ioloop.IOLoop.instance().start() + + def _create_tornado_web_app(self): + class TabPyTornadoApp(tornado.web.Application): + is_closing = False + + def signal_handler(self, signal, _): + logger.critical(f"Exiting on signal {signal}...") + self.is_closing = True + + def try_exit(self): + if self.is_closing: + tornado.ioloop.IOLoop.instance().stop() + logger.info("Shutting down TabPy...") + + logger.info("Initializing TabPy...") + tornado.ioloop.IOLoop.instance().run_sync( + lambda: init_ps_server(self.settings, self.tabpy_state) + ) + logger.info("Done initializing TabPy.") + + executor = concurrent.futures.ThreadPoolExecutor( + max_workers=multiprocessing.cpu_count() + ) + + # initialize Tornado application + _init_asyncio_patch() + application = TabPyTornadoApp( + [ + # skip MainHandler to use StaticFileHandler .* page requests and + # default to index.html + # (r"/", MainHandler), + ( + self.subdirectory + r"/query/([^/]+)", + QueryPlaneHandler, + dict(app=self), + ), + (self.subdirectory + r"/status", StatusHandler, dict(app=self)), + (self.subdirectory + r"/info", ServiceInfoHandler, dict(app=self)), + (self.subdirectory + r"/endpoints", EndpointsHandler, dict(app=self)), + ( + self.subdirectory + r"/endpoints/([^/]+)?", + EndpointHandler, + dict(app=self), + ), + ( + self.subdirectory + r"/evaluate", + EvaluationPlaneHandler, + dict(executor=executor, app=self), + ), + ( + self.subdirectory + r"/configurations/endpoint_upload_destination", + UploadDestinationHandler, + dict(app=self), + ), + ( + self.subdirectory + r"/(.*)", + tornado.web.StaticFileHandler, + dict( + path=self.settings[SettingsParameters.StaticPath], + default_filename="index.html", + ), + ), + ], + debug=False, + **self.settings, + ) + + signal.signal(signal.SIGINT, application.signal_handler) + tornado.ioloop.PeriodicCallback(application.try_exit, 500).start() + + signal.signal(signal.SIGINT, application.signal_handler) + tornado.ioloop.PeriodicCallback(application.try_exit, 500).start() + + return application + + def _set_parameter(self, parser, settings_key, config_key, default_val, parse_function): + key_is_set = False + + if ( + config_key is not None + and parser.has_section("TabPy") + and parser.has_option("TabPy", config_key) + ): + if parse_function is None: + parse_function = parser.get + self.settings[settings_key] = parse_function("TabPy", config_key) + key_is_set = True + logger.debug( + f"Parameter {settings_key} set to " + f'"{self.settings[settings_key]}" ' + "from config file or environment variable" + ) + + if not key_is_set and default_val is not None: + self.settings[settings_key] = default_val + key_is_set = True + logger.debug( + f"Parameter {settings_key} set to " + f'"{self.settings[settings_key]}" ' + "from default value" + ) + + if not key_is_set: + logger.debug(f"Parameter {settings_key} is not set") + + def _parse_config(self, config_file): + """Provide consistent mechanism for pulling in configuration. + + Attempt to retain backward compatibility for + existing implementations by grabbing port + setting from CLI first. + + Take settings in the following order: + + 1. CLI arguments if present + 2. config file + 3. OS environment variables (for ease of + setting defaults if not present) + 4. current defaults if a setting is not present in any location + + Additionally provide similar configuration capabilities in between + config file and environment variables. + For consistency use the same variable name in the config file as + in the os environment. + For naming standards use all capitals and start with 'TABPY_' + """ + self.settings = {} + self.subdirectory = "" + self.tabpy_state = None + self.python_service = None + self.credentials = {} + + pkg_path = os.path.dirname(tabpy.__file__) + + parser = configparser.ConfigParser(os.environ) + + if os.path.isfile(config_file): + with open(config_file) as f: + parser.read_string(f.read()) + else: + logger.warning( + f"Unable to find config file at {config_file}, " + "using default settings." + ) + + settings_parameters = [ + (SettingsParameters.Port, ConfigParameters.TABPY_PORT, 9004, None), + (SettingsParameters.ServerVersion, None, __version__, None), + (SettingsParameters.EvaluateTimeout, ConfigParameters.TABPY_EVALUATE_TIMEOUT, + 30, parser.getfloat), + (SettingsParameters.UploadDir, ConfigParameters.TABPY_QUERY_OBJECT_PATH, + os.path.join(pkg_path, "tmp", "query_objects"), None), + (SettingsParameters.TransferProtocol, ConfigParameters.TABPY_TRANSFER_PROTOCOL, + "http", None), + (SettingsParameters.CertificateFile, ConfigParameters.TABPY_CERTIFICATE_FILE, + None, None), + (SettingsParameters.KeyFile, ConfigParameters.TABPY_KEY_FILE, None, None), + (SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, + os.path.join(pkg_path, "server"), None), + (SettingsParameters.StaticPath, ConfigParameters.TABPY_STATIC_PATH, + os.path.join(pkg_path, "server", "static"), None), + (ConfigParameters.TABPY_PWD_FILE, ConfigParameters.TABPY_PWD_FILE, None, None), + (SettingsParameters.LogRequestContext, ConfigParameters.TABPY_LOG_DETAILS, + "false", None), + (SettingsParameters.MaxRequestSizeInMb, ConfigParameters.TABPY_MAX_REQUEST_SIZE_MB, + 100, None), + ] + + for setting, parameter, default_val, parse_function in settings_parameters: + self._set_parameter(parser, setting, parameter, default_val, parse_function) + + if not os.path.exists(self.settings[SettingsParameters.UploadDir]): + os.makedirs(self.settings[SettingsParameters.UploadDir]) + + # set and validate transfer protocol + self.settings[SettingsParameters.TransferProtocol] = self.settings[ + SettingsParameters.TransferProtocol + ].lower() + + self._validate_transfer_protocol_settings() + + # if state.ini does not exist try and create it - remove + # last dependence on batch/shell script + self.settings[SettingsParameters.StateFilePath] = os.path.realpath( + os.path.normpath( + os.path.expanduser(self.settings[SettingsParameters.StateFilePath]) + ) + ) + state_config, self.tabpy_state = self._build_tabpy_state() + + self.python_service = PythonServiceHandler(PythonService()) + self.settings["compress_response"] = True + self.settings[SettingsParameters.StaticPath] = os.path.abspath( + self.settings[SettingsParameters.StaticPath] + ) + logger.debug( + f"Static pages folder set to " + f'"{self.settings[SettingsParameters.StaticPath]}"' + ) + + # Set subdirectory from config if applicable + if state_config.has_option("Service Info", "Subdirectory"): + self.subdirectory = "/" + state_config.get("Service Info", "Subdirectory") + + # If passwords file specified load credentials + if ConfigParameters.TABPY_PWD_FILE in self.settings: + if not self._parse_pwd_file(): + msg = ( + "Failed to read passwords file " + f"{self.settings[ConfigParameters.TABPY_PWD_FILE]}" + ) + logger.critical(msg) + raise RuntimeError(msg) + else: + logger.info( + "Password file is not specified: " "Authentication is not enabled" + ) + + features = self._get_features() + self.settings[SettingsParameters.ApiVersions] = {"v1": {"features": features}} + + self.settings[SettingsParameters.LogRequestContext] = ( + self.settings[SettingsParameters.LogRequestContext].lower() != "false" + ) + call_context_state = ( + "enabled" + if self.settings[SettingsParameters.LogRequestContext] + else "disabled" + ) + logger.info(f"Call context logging is {call_context_state}") + + def _validate_transfer_protocol_settings(self): + if SettingsParameters.TransferProtocol not in self.settings: + msg = "Missing transfer protocol information." + logger.critical(msg) + raise RuntimeError(msg) + + protocol = self.settings[SettingsParameters.TransferProtocol] + + if protocol == "http": + return + + if protocol != "https": + msg = f"Unsupported transfer protocol: {protocol}" + logger.critical(msg) + raise RuntimeError(msg) + + self._validate_cert_key_state( + "The parameter(s) {} must be set.", + SettingsParameters.CertificateFile in self.settings, + SettingsParameters.KeyFile in self.settings, + ) + cert = self.settings[SettingsParameters.CertificateFile] + + self._validate_cert_key_state( + "The parameter(s) {} must point to " "an existing file.", + os.path.isfile(cert), + os.path.isfile(self.settings[SettingsParameters.KeyFile]), + ) + tabpy.tabpy_server.app.util.validate_cert(cert) + + @staticmethod + def _validate_cert_key_state(msg, cert_valid, key_valid): + cert_and_key_param = ( + f"{ConfigParameters.TABPY_CERTIFICATE_FILE} and " + f"{ConfigParameters.TABPY_KEY_FILE}" + ) + https_error = "Error using HTTPS: " + err = None + if not cert_valid and not key_valid: + err = https_error + msg.format(cert_and_key_param) + elif not cert_valid: + err = https_error + msg.format(ConfigParameters.TABPY_CERTIFICATE_FILE) + elif not key_valid: + err = https_error + msg.format(ConfigParameters.TABPY_KEY_FILE) + + if err is not None: + logger.critical(err) + raise RuntimeError(err) + + def _parse_pwd_file(self): + succeeded, self.credentials = parse_pwd_file( + self.settings[ConfigParameters.TABPY_PWD_FILE] + ) + + if succeeded and len(self.credentials) == 0: + logger.error("No credentials found") + succeeded = False + + return succeeded + + def _get_features(self): + features = {} + + # Check for auth + if ConfigParameters.TABPY_PWD_FILE in self.settings: + features["authentication"] = { + "required": True, + "methods": {"basic-auth": {}}, + } + + return features + + def _build_tabpy_state(self): + pkg_path = os.path.dirname(tabpy.__file__) + state_file_dir = self.settings[SettingsParameters.StateFilePath] + state_file_path = os.path.join(state_file_dir, "state.ini") + if not os.path.isfile(state_file_path): + state_file_template_path = os.path.join( + pkg_path, "server", "state.ini.template" + ) + logger.debug( + f"File {state_file_path} not found, creating from " + f"template {state_file_template_path}..." + ) + shutil.copy(state_file_template_path, state_file_path) + + logger.info(f"Loading state from state file {state_file_path}") + tabpy_state = _get_state_from_file(state_file_dir) + return tabpy_state, TabPyState(config=tabpy_state, settings=self.settings) diff --git a/tabpy/tabpy_server/app/util.py b/tabpy/tabpy_server/app/util.py new file mode 100644 index 00000000..944b5997 --- /dev/null +++ b/tabpy/tabpy_server/app/util.py @@ -0,0 +1,88 @@ +import csv +from datetime import datetime +import logging +from OpenSSL import crypto +import os + + +logger = logging.getLogger(__name__) + + +def validate_cert(cert_file_path): + with open(cert_file_path, "r") as f: + cert_buf = f.read() + + cert = crypto.load_certificate(crypto.FILETYPE_PEM, cert_buf) + + date_format, encoding = "%Y%m%d%H%M%SZ", "ascii" + not_before = datetime.strptime(cert.get_notBefore().decode(encoding), date_format) + not_after = datetime.strptime(cert.get_notAfter().decode(encoding), date_format) + now = datetime.now() + + https_error = "Error using HTTPS: " + if now < not_before: + msg = https_error + f"The certificate provided is not valid until {not_before}." + logger.critical(msg) + raise RuntimeError(msg) + if now > not_after: + msg = https_error + f"The certificate provided expired on {not_after}." + logger.critical(msg) + raise RuntimeError(msg) + + +def parse_pwd_file(pwd_file_name): + """ + Parses passwords file and returns set of credentials. + + Parameters + ---------- + pwd_file_name : str + Passwords file name. + + Returns + ------- + succeeded : bool + True if specified file was parsed successfully. + False if there were any issues with parsing specified file. + + credentials : dict + Credentials from the file. Empty if succeeded is False. + """ + logger.info(f"Parsing passwords file {pwd_file_name}...") + + if not os.path.isfile(pwd_file_name): + logger.critical(f"Passwords file {pwd_file_name} not found") + return False, {} + + credentials = {} + with open(pwd_file_name) as pwd_file: + pwd_file_reader = csv.reader(pwd_file, delimiter=" ") + for row in pwd_file_reader: + # skip empty lines + if len(row) == 0: + continue + + # skip commented lines + if row[0][0] == "#": + continue + + if len(row) != 2: + logger.error(f'Incorrect entry "{row}" in password file') + return False, {} + + login = row[0].lower() + if login in credentials: + logger.error( + f"Multiple entries for username {login} in password file" + ) + return False, {} + + if len(row[1]) > 0: + credentials[login] = row[1] + logger.debug(f"Found username {login}") + else: + logger.warning(f"Found username {row[0]} but no password") + return False, {} + + logger.info("Authentication is enabled") + return True, credentials diff --git a/tabpy/tabpy_server/common/__init__.py b/tabpy/tabpy_server/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf new file mode 100644 index 00000000..80e9dd49 --- /dev/null +++ b/tabpy/tabpy_server/common/default.conf @@ -0,0 +1,62 @@ +[TabPy] +# TABPY_QUERY_OBJECT_PATH = /tmp/query_objects +# TABPY_PORT = 9004 +# TABPY_STATE_PATH = ./tabpy/server + +# Where static pages live +# TABPY_STATIC_PATH = ./tabpy/server/static + +# For how to configure TabPy authentication read +# Authentication section in docs/server-config.md. +# TABPY_PWD_FILE = /path/to/password/file.txt + +# To set up secure TabPy uncomment and modify the following lines. +# Note only PEM-encoded x509 certificates are supported. +# TABPY_TRANSFER_PROTOCOL = https +# TABPY_CERTIFICATE_FILE = /path/to/certificate/file.crt +# TABPY_KEY_FILE = /path/to/key/file.key + +# Log additional request details including caller IP, full URL, client +# end user info if provided. +# TABPY_LOG_DETAILS = true + +# Limit request size (in Mb) - any request which size exceeds +# specified amount will be rejected by TabPy. +# Default value is 100 Mb. +# TABPY_MAX_REQUEST_SIZE_MB = 100 + +# Configure how long a custom script provided to the /evaluate method +# will run before throwing a TimeoutError. +# The value should be a float representing the timeout time in seconds. +# TABPY_EVALUATE_TIMEOUT = 30 + +[loggers] +keys=root + +[handlers] +keys=rootHandler,rotatingFileHandler + +[formatters] +keys=rootFormatter + +[logger_root] +level=DEBUG +handlers=rootHandler,rotatingFileHandler +qualname=root +propagete=0 + +[handler_rootHandler] +class=StreamHandler +level=DEBUG +formatter=rootFormatter +args=(sys.stdout,) + +[handler_rotatingFileHandler] +class=handlers.RotatingFileHandler +level=DEBUG +formatter=rootFormatter +args=('tabpy_log.log', 'a', 1000000, 5) + +[formatter_rootFormatter] +format=%(asctime)s [%(levelname)s] (%(filename)s:%(module)s:%(lineno)d): %(message)s +datefmt=%Y-%m-%d,%H:%M:%S diff --git a/tabpy/tabpy_server/common/endpoint_file_mgr.py b/tabpy/tabpy_server/common/endpoint_file_mgr.py new file mode 100644 index 00000000..6b7fed00 --- /dev/null +++ b/tabpy/tabpy_server/common/endpoint_file_mgr.py @@ -0,0 +1,94 @@ +""" +This module provides functionality required for managing endpoint objects in +TabPy. It provides a way to download endpoint files from remote +and then properly cleanup local the endpoint files on update/remove of endpoint +objects. + +The local temporary files for TabPy will by default located at + /tmp/query_objects + +""" +import logging +import os +import shutil +from re import compile as _compile + + +_name_checker = _compile(r"^[a-zA-Z0-9-_\s]+$") + + +def _check_endpoint_name(name, logger=logging.getLogger(__name__)): + """Checks that the endpoint name is valid by comparing it with an RE and + checking that it is not reserved.""" + if not isinstance(name, str): + msg = "Endpoint name must be a string" + logger.log(logging.CRITICAL, msg) + raise TypeError(msg) + + if name == "": + msg = "Endpoint name cannot be empty" + logger.log(logging.CRITICAL, msg) + raise ValueError(msg) + + if not _name_checker.match(name): + msg = ( + "Endpoint name can only contain: a-z, A-Z, 0-9," + " underscore, hyphens and spaces." + ) + logger.log(logging.CRITICAL, msg) + raise ValueError(msg) + + +def grab_files(directory): + """ + Generator that returns all files in a directory. + """ + if not os.path.isdir(directory): + return + else: + for name in os.listdir(directory): + full_path = os.path.join(directory, name) + if os.path.isdir(full_path): + for entry in grab_files(full_path): + yield entry + elif os.path.isfile(full_path): + yield full_path + + +def cleanup_endpoint_files( + name, query_path, logger=logging.getLogger(__name__), retain_versions=None +): + """ + Cleanup the disk space a certain endpiont uses. + + Parameters + ---------- + name : str + The endpoint name + + retain_version : int, optional + If given, then all files for this endpoint are removed except the + folder for the given version, otherwise, all files for that endpoint + are removed. + """ + _check_endpoint_name(name, logger=logger) + local_dir = os.path.join(query_path, name) + + # nothing to clean, this is true for state file path where we load + # Query Object directly from the state path instead of downloading + # to temporary location + if not os.path.exists(local_dir): + return + + if not retain_versions: + shutil.rmtree(local_dir) + else: + retain_folders = [ + os.path.join(local_dir, str(version)) for version in retain_versions + ] + logger.log(logging.INFO, f"Retain folders: {retain_folders}") + + for file_or_dir in os.listdir(local_dir): + candidate_dir = os.path.join(local_dir, file_or_dir) + if os.path.isdir(candidate_dir) and (candidate_dir not in retain_folders): + shutil.rmtree(candidate_dir) diff --git a/tabpy/tabpy_server/common/messages.py b/tabpy/tabpy_server/common/messages.py new file mode 100644 index 00000000..ad684319 --- /dev/null +++ b/tabpy/tabpy_server/common/messages.py @@ -0,0 +1,172 @@ +import abc +from abc import ABCMeta +from collections import namedtuple +import json + + +class Msg: + """ + An abstract base class for all messages used for communicating between + the WebServices. + + The minimal functionality is the ability to instantiate a Msg from JSON + and to write a Msg instance to JSON. + + We use namedtuples because they are lightweight and immutable. The splat + operator (*) that we inherit from namedtuple is also convenient. We empty + __slots__ to avoid unnecessary overhead. + """ + + __metaclass__ = ABCMeta + + @abc.abstractmethod + def for_json(self): + d = self._asdict() + type_str = self.__class__.__name__ + d.update({"type": type_str}) + return d + + @abc.abstractmethod + def to_json(self): + return json.dumps(self.for_json()) + + @staticmethod + def from_json(str): + d = json.loads(str) + type_str = d["type"] + del d["type"] + return eval(type_str)(**d) + + +class LoadSuccessful( + namedtuple( + "LoadSuccessful", ["uri", "path", "version", "is_update", "endpoint_type"] + ), + Msg, +): + __slots__ = () + + +class LoadFailed(namedtuple("LoadFailed", ["uri", "version", "error_msg"]), Msg): + __slots__ = () + + +class LoadInProgress( + namedtuple( + "LoadInProgress", ["uri", "path", "version", "is_update", "endpoint_type"] + ), + Msg, +): + __slots__ = () + + +class Query(namedtuple("Query", ["uri", "params"]), Msg): + __slots__ = () + + +class QuerySuccessful( + namedtuple("QuerySuccessful", ["uri", "version", "response"]), Msg +): + __slots__ = () + + +class LoadObject( + namedtuple("LoadObject", ["uri", "url", "version", "is_update", "endpoint_type"]), + Msg, +): + __slots__ = () + + +class DeleteObjects(namedtuple("DeleteObjects", ["uris"]), Msg): + __slots__ = () + + +# Used for testing to flush out objects +class FlushObjects(namedtuple("FlushObjects", []), Msg): + __slots__ = () + + +class ObjectsDeleted(namedtuple("ObjectsDeleted", ["uris"]), Msg): + __slots__ = () + + +class ObjectsFlushed(namedtuple("ObjectsFlushed", ["n_before", "n_after"]), Msg): + __slots__ = () + + +class CountObjects(namedtuple("CountObjects", []), Msg): + __slots__ = () + + +class ObjectCount(namedtuple("ObjectCount", ["count"]), Msg): + __slots__ = () + + +class ListObjects(namedtuple("ListObjects", []), Msg): + __slots__ = () + + +class ObjectList(namedtuple("ObjectList", ["objects"]), Msg): + __slots__ = () + + +class UnknownURI(namedtuple("UnknownURI", ["uri"]), Msg): + __slots__ = () + + +class UnknownMessage(namedtuple("UnknownMessage", ["msg"]), Msg): + __slots__ = () + + +class DownloadSkipped( + namedtuple("DownloadSkipped", ["uri", "version", "msg", "host"]), Msg +): + __slots__ = () + + +class QueryFailed(namedtuple("QueryFailed", ["uri", "error"]), Msg): + __slots__ = () + + +class QueryError(namedtuple("QueryError", ["uri", "error"]), Msg): + __slots__ = () + + +class CheckHealth(namedtuple("CheckHealth", []), Msg): + __slots__ = () + + +class Healthy(namedtuple("Healthy", []), Msg): + __slots__ = () + + +class Unhealthy(namedtuple("Unhealthy", []), Msg): + __slots__ = () + + +class Ping(namedtuple("Ping", ["id"]), Msg): + __slots__ = () + + +class Pong(namedtuple("Pong", ["id"]), Msg): + __slots__ = () + + +class Listening(namedtuple("Listening", []), Msg): + __slots__ = () + + +class EngineFailure(namedtuple("EngineFailure", ["error"]), Msg): + __slots__ = () + + +class FlushLogs(namedtuple("FlushLogs", []), Msg): + __slots__ = () + + +class LogsFlushed(namedtuple("LogsFlushed", []), Msg): + __slots__ = () + + +class ServiceError(namedtuple("ServiceError", ["error"]), Msg): + __slots__ = () diff --git a/tabpy/tabpy_server/common/util.py b/tabpy/tabpy_server/common/util.py new file mode 100644 index 00000000..c731450a --- /dev/null +++ b/tabpy/tabpy_server/common/util.py @@ -0,0 +1,3 @@ +def format_exception(e, context): + err_msg = f"{e.__class__.__name__} : {str(e)}" + return err_msg diff --git a/tabpy/tabpy_server/handlers/__init__.py b/tabpy/tabpy_server/handlers/__init__.py new file mode 100644 index 00000000..0c00cde6 --- /dev/null +++ b/tabpy/tabpy_server/handlers/__init__.py @@ -0,0 +1,13 @@ +from tabpy.tabpy_server.handlers.base_handler import BaseHandler +from tabpy.tabpy_server.handlers.main_handler import MainHandler +from tabpy.tabpy_server.handlers.management_handler import ManagementHandler + +from tabpy.tabpy_server.handlers.endpoint_handler import EndpointHandler +from tabpy.tabpy_server.handlers.endpoints_handler import EndpointsHandler +from tabpy.tabpy_server.handlers.evaluation_plane_handler import EvaluationPlaneHandler +from tabpy.tabpy_server.handlers.query_plane_handler import QueryPlaneHandler +from tabpy.tabpy_server.handlers.service_info_handler import ServiceInfoHandler +from tabpy.tabpy_server.handlers.status_handler import StatusHandler +from tabpy.tabpy_server.handlers.upload_destination_handler import ( + UploadDestinationHandler, +) diff --git a/tabpy/tabpy_server/handlers/base_handler.py b/tabpy/tabpy_server/handlers/base_handler.py new file mode 100644 index 00000000..5ba55546 --- /dev/null +++ b/tabpy/tabpy_server/handlers/base_handler.py @@ -0,0 +1,425 @@ +import base64 +import binascii +import concurrent +import json +import logging +import tornado.web +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers.util import hash_password +import uuid + + +STAGING_THREAD = concurrent.futures.ThreadPoolExecutor(max_workers=3) + + +class ContextLoggerWrapper: + """ + This class appends request context to logged messages. + """ + + @staticmethod + def _generate_call_id(): + return str(uuid.uuid4()) + + def __init__(self, request: tornado.httputil.HTTPServerRequest): + self.call_id = self._generate_call_id() + self.set_request(request) + + self.tabpy_username = None + self.log_request_context = False + self.request_context_logged = False + + def set_request(self, request: tornado.httputil.HTTPServerRequest): + """ + Set HTTP(S) request for logger. Headers will be used to + append request data as client information, Tableau user name, etc. + """ + self.remote_ip = request.remote_ip + self.method = request.method + self.url = request.full_url() + + self.client = request.headers.get("TabPy-Client", None) + self.tableau_username = request.headers.get("TabPy-User", None) + + def set_tabpy_username(self, tabpy_username: str): + self.tabpy_username = tabpy_username + + def enable_context_logging(self, enable: bool): + """ + Enable/disable request context information logging. + + Parameters + ---------- + enable: bool + If True request context information will be logged and + every log entry for a request handler will have call ID + with it. + """ + self.log_request_context = enable + + def _log_context_info(self): + if not self.log_request_context: + return + + context = f"Call ID: {self.call_id}" + + if self.remote_ip is not None: + context += f", Caller: {self.remote_ip}" + + if self.method is not None: + context += f", Method: {self.method}" + + if self.url is not None: + context += f", URL: {self.url}" + + if self.client is not None: + context += f", Client: {self.client}" + + if self.tableau_username is not None: + context += f", Tableau user: {self.tableau_username}" + + if self.tabpy_username is not None: + context += f", TabPy user: {self.tabpy_username}" + + logging.getLogger(__name__).log(logging.INFO, context) + self.request_context_logged = True + + def log(self, level: int, msg: str): + """ + Log message with or without call ID. If call context is logged and + call ID added to any log entry is specified by if context logging + is enabled (see CallContext.enable_context_logging for more details). + + Parameters + ---------- + level: int + Log level: logging.CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET. + + msg: str + Message format string. + + args + Same as args in Logger.debug(). + + kwargs + Same as kwargs in Logger.debug(). + """ + extended_msg = msg + if self.log_request_context: + if not self.request_context_logged: + self._log_context_info() + + extended_msg += f", <>" + + logging.getLogger(__name__).log(level, extended_msg) + + +class BaseHandler(tornado.web.RequestHandler): + def initialize(self, app): + self.tabpy_state = app.tabpy_state + # set content type to application/json + self.set_header("Content-Type", "application/json") + self.protocol = self.settings[SettingsParameters.TransferProtocol] + self.port = self.settings[SettingsParameters.Port] + self.python_service = app.python_service + self.credentials = app.credentials + self.username = None + self.password = None + self.eval_timeout = self.settings[SettingsParameters.EvaluateTimeout] + + self.logger = ContextLoggerWrapper(self.request) + self.logger.enable_context_logging( + app.settings[SettingsParameters.LogRequestContext] + ) + self.logger.log(logging.DEBUG, "Checking if need to handle authentication") + self.not_authorized = not self.handle_authentication("v1") + + def error_out(self, code, log_message, info=None): + self.set_status(code) + self.write(json.dumps({"message": log_message, "info": info or {}})) + + # We want to duplicate error message in console for + # loggers are misconfigured or causing the failure + # themselves + print(info) + self.logger.log( + logging.ERROR, + 'Responding with status={}, message="{}", info="{}"'.format( + code, log_message, info + ), + ) + self.finish() + + def options(self): + # add CORS headers if TabPy has a cors_origin specified + self._add_CORS_header() + self.write({}) + + def _add_CORS_header(self): + """ + Add CORS header if the TabPy has attribute _cors_origin + and _cors_origin is not an empty string. + """ + origin = self.tabpy_state.get_access_control_allow_origin() + if len(origin) > 0: + self.set_header("Access-Control-Allow-Origin", origin) + self.logger.log(logging.DEBUG, f"Access-Control-Allow-Origin:{origin}") + + headers = self.tabpy_state.get_access_control_allow_headers() + if len(headers) > 0: + self.set_header("Access-Control-Allow-Headers", headers) + self.logger.log(logging.DEBUG, f"Access-Control-Allow-Headers:{headers}") + + methods = self.tabpy_state.get_access_control_allow_methods() + if len(methods) > 0: + self.set_header("Access-Control-Allow-Methods", methods) + self.logger.log(logging.DEBUG, f"Access-Control-Allow-Methods:{methods}") + + def _get_auth_method(self, api_version) -> (bool, str): + """ + Finds authentication method if provided. + + Parameters + ---------- + api_version : str + API version for authentication. + + Returns + ------- + bool + True if known authentication method is found. + False otherwise. + + str + Name of authentication method used by client. + If empty no authentication required. + + (True, '') as result of this function means authentication + is not needed. + """ + if api_version not in self.settings[SettingsParameters.ApiVersions]: + self.logger.log(logging.CRITICAL, f'Unknown API version "{api_version}"') + return False, "" + + version_settings = self.settings[SettingsParameters.ApiVersions][api_version] + if "features" not in version_settings: + self.logger.log( + logging.INFO, f'No features configured for API "{api_version}"' + ) + return True, "" + + features = version_settings["features"] + if ( + "authentication" not in features + or not features["authentication"]["required"] + ): + self.logger.log( + logging.INFO, + "Authentication is not a required feature for API " f'"{api_version}"', + ) + return True, "" + + auth_feature = features["authentication"] + if "methods" not in auth_feature: + self.logger.log( + logging.INFO, + "Authentication method is not configured for API " f'"{api_version}"', + ) + + methods = auth_feature["methods"] + if "basic-auth" in auth_feature["methods"]: + return True, "basic-auth" + # Add new methods here... + + # No known methods were found + self.logger.log( + logging.CRITICAL, + f'Unknown authentication method(s) "{methods}" are configured ' + f'for API "{api_version}"', + ) + return False, "" + + def _get_basic_auth_credentials(self) -> bool: + """ + Find credentials for basic access authentication method. Credentials if + found stored in Credentials.username and Credentials.password. + + Returns + ------- + bool + True if valid credentials were found. + False otherwise. + """ + self.logger.log( + logging.DEBUG, "Checking request headers for authentication data" + ) + if "Authorization" not in self.request.headers: + self.logger.log(logging.INFO, "Authorization header not found") + return False + + auth_header = self.request.headers["Authorization"] + auth_header_list = auth_header.split(" ") + if len(auth_header_list) != 2 or auth_header_list[0] != "Basic": + self.logger.log( + logging.ERROR, f'Unknown authentication method "{auth_header}"' + ) + return False + + try: + cred = base64.b64decode(auth_header_list[1]).decode("utf-8") + except (binascii.Error, UnicodeDecodeError) as ex: + self.logger.log(logging.CRITICAL, f"Cannot decode credentials: {str(ex)}") + return False + + login_pwd = cred.split(":") + if len(login_pwd) != 2: + self.logger.log(logging.ERROR, "Invalid string in encoded credentials") + return False + + self.username = login_pwd[0] + self.logger.set_tabpy_username(self.username) + self.password = login_pwd[1] + return True + + def _get_credentials(self, method) -> bool: + """ + Find credentials for specified authentication method. Credentials if + found stored in self.username and self.password. + + Parameters + ---------- + method: str + Authentication method name. + + Returns + ------- + bool + True if valid credentials were found. + False otherwise. + """ + if method == "basic-auth": + return self._get_basic_auth_credentials() + # Add new methods here... + + # No known methods were found + self.logger.log( + logging.CRITICAL, + f'Unknown authentication method(s) "{method}" are configured ', + ) + return False + + def _validate_basic_auth_credentials(self) -> bool: + """ + Validates username:pwd if they are the same as + stored credentials. + + Returns + ------- + bool + True if credentials has key login and + credentials[login] equal SHA3(pwd), False + otherwise. + """ + login = self.username.lower() + self.logger.log( + logging.DEBUG, f'Validating credentials for user name "{login}"' + ) + if login not in self.credentials: + self.logger.log(logging.ERROR, f'User name "{self.username}" not found') + return False + + hashed_pwd = hash_password(login, self.password) + if self.credentials[login].lower() != hashed_pwd.lower(): + self.logger.log( + logging.ERROR, f'Wrong password for user name "{self.username}"' + ) + return False + + return True + + def _validate_credentials(self, method) -> bool: + """ + Validates credentials according to specified methods if they + are what expected. + + Parameters + ---------- + method: str + Authentication method name. + + Returns + ------- + bool + True if credentials are valid. + False otherwise. + """ + if method == "basic-auth": + return self._validate_basic_auth_credentials() + # Add new methods here... + + # No known methods were found + self.logger.log( + logging.CRITICAL, + f'Unknown authentication method(s) "{method}" are configured ', + ) + return False + + def handle_authentication(self, api_version) -> bool: + """ + If authentication feature is configured checks provided + credentials. + + Parameters + ---------- + api_version : str + API version for authentication. + + Returns + ------- + bool + True if authentication is not required. + True if authentication is required and valid + credentials provided. + False otherwise. + """ + self.logger.log(logging.DEBUG, "Handling authentication") + found, method = self._get_auth_method(api_version) + if not found: + return False + + if method == "": + # Do not validate credentials + return True + + if not self._get_credentials(method): + return False + + return self._validate_credentials(method) + + def should_fail_with_not_authorized(self): + """ + Checks if authentication is required: + - if it is not returns false, None + - if it is required validates provided credentials + + Returns + ------- + bool + False if authentication is not required or is + required and validation for credentials passes. + True if validation for credentials failed. + """ + return self.not_authorized + + def fail_with_not_authorized(self): + """ + Prepares server 401 response. + """ + self.logger.log(logging.ERROR, "Failing with 401 for unauthorized request") + self.set_status(401) + self.set_header("WWW-Authenticate", f'Basic realm="{self.tabpy_state.name}"') + self.error_out( + 401, + info="Unauthorized request.", + log_message="Invalid credentials provided.", + ) diff --git a/tabpy/tabpy_server/handlers/endpoint_handler.py b/tabpy/tabpy_server/handlers/endpoint_handler.py new file mode 100644 index 00000000..022d8e0b --- /dev/null +++ b/tabpy/tabpy_server/handlers/endpoint_handler.py @@ -0,0 +1,141 @@ +""" +HTTP handeler to serve specific endpoint request like +http://myserver:9004/endpoints/mymodel + +For how generic endpoints requests is served look +at endpoints_handler.py +""" + +import json +import logging +import shutil +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.handlers import ManagementHandler +from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD +from tabpy.tabpy_server.management.state import get_query_object_path +from tabpy.tabpy_server.psws.callbacks import on_state_change +from tornado import gen + + +class EndpointHandler(ManagementHandler): + def initialize(self, app): + super(EndpointHandler, self).initialize(app) + + def get(self, endpoint_name): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + self.logger.log(logging.DEBUG, f"Processing GET for /endpoints/{endpoint_name}") + + self._add_CORS_header() + if not endpoint_name: + self.write(json.dumps(self.tabpy_state.get_endpoints())) + else: + if endpoint_name in self.tabpy_state.get_endpoints(): + self.write(json.dumps(self.tabpy_state.get_endpoints()[endpoint_name])) + else: + self.error_out( + 404, + "Unknown endpoint", + info=f"Endpoint {endpoint_name} is not found", + ) + + @gen.coroutine + def put(self, name): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + self.logger.log(logging.DEBUG, f"Processing PUT for /endpoints/{name}") + + try: + if not self.request.body: + self.error_out(400, "Input body cannot be empty") + self.finish() + return + try: + request_data = json.loads(self.request.body.decode("utf-8")) + except BaseException as ex: + self.error_out( + 400, log_message="Failed to decode input body", info=str(ex) + ) + self.finish() + return + + # check if endpoint exists + endpoints = self.tabpy_state.get_endpoints(name) + if len(endpoints) == 0: + self.error_out(404, f"endpoint {name} does not exist.") + self.finish() + return + + new_version = int(endpoints[name]["version"]) + 1 + self.logger.log(logging.INFO, f"Endpoint info: {request_data}") + err_msg = yield self._add_or_update_endpoint( + "update", name, new_version, request_data + ) + if err_msg: + self.error_out(400, err_msg) + self.finish() + else: + self.write(self.tabpy_state.get_endpoints(name)) + self.finish() + + except Exception as e: + err_msg = format_exception(e, "update_endpoint") + self.error_out(500, err_msg) + self.finish() + + @gen.coroutine + def delete(self, name): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + self.logger.log(logging.DEBUG, f"Processing DELETE for /endpoints/{name}") + + try: + endpoints = self.tabpy_state.get_endpoints(name) + if len(endpoints) == 0: + self.error_out(404, f"endpoint {name} does not exist.") + self.finish() + return + + # update state + try: + endpoint_info = self.tabpy_state.delete_endpoint(name) + except Exception as e: + self.error_out(400, f"Error when removing endpoint: {e.message}") + self.finish() + return + + # delete files + if endpoint_info["type"] != "alias": + delete_path = get_query_object_path( + self.settings["state_file_path"], name, None + ) + try: + yield self._delete_po_future(delete_path) + except Exception as e: + self.error_out(400, f"Error while deleting: {e}") + self.finish() + return + + self.set_status(204) + self.finish() + + except Exception as e: + err_msg = format_exception(e, "delete endpoint") + self.error_out(500, err_msg) + self.finish() + + on_state_change( + self.settings, self.tabpy_state, self.python_service, self.logger + ) + + @gen.coroutine + def _delete_po_future(self, delete_path): + future = STAGING_THREAD.submit(shutil.rmtree, delete_path) + ret = yield future + raise gen.Return(ret) diff --git a/tabpy/tabpy_server/handlers/endpoints_handler.py b/tabpy/tabpy_server/handlers/endpoints_handler.py new file mode 100644 index 00000000..66132dd2 --- /dev/null +++ b/tabpy/tabpy_server/handlers/endpoints_handler.py @@ -0,0 +1,75 @@ +""" +HTTP handeler to serve general endpoints request, specifically +http://myserver:9004/endpoints + +For how individual endpoint requests are served look +at endpoint_handler.py +""" + +import json +import logging +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.handlers import ManagementHandler +from tornado import gen + + +class EndpointsHandler(ManagementHandler): + def initialize(self, app): + super(EndpointsHandler, self).initialize(app) + + def get(self): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + self._add_CORS_header() + self.write(json.dumps(self.tabpy_state.get_endpoints())) + + @gen.coroutine + def post(self): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + try: + if not self.request.body: + self.error_out(400, "Input body cannot be empty") + self.finish() + return + + try: + request_data = json.loads(self.request.body.decode("utf-8")) + except Exception as ex: + self.error_out(400, "Failed to decode input body", str(ex)) + self.finish() + return + + if "name" not in request_data: + self.error_out(400, "name is required to add an endpoint.") + self.finish() + return + + name = request_data["name"] + + # check if endpoint already exist + if name in self.tabpy_state.get_endpoints(): + self.error_out(400, f"endpoint {name} already exists.") + self.finish() + return + + self.logger.log(logging.DEBUG, f'Adding endpoint "{name}"') + err_msg = yield self._add_or_update_endpoint("add", name, 1, request_data) + if err_msg: + self.error_out(400, err_msg) + else: + self.logger.log(logging.DEBUG, f"Endpoint {name} successfully added") + self.set_status(201) + self.write(self.tabpy_state.get_endpoints(name)) + self.finish() + return + + except Exception as e: + err_msg = format_exception(e, "/add_endpoint") + self.error_out(500, "error adding endpoint", err_msg) + self.finish() + return diff --git a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py new file mode 100644 index 00000000..13ee1175 --- /dev/null +++ b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py @@ -0,0 +1,142 @@ +from tabpy.tabpy_server.handlers import BaseHandler +import json +import simplejson +import logging +from tabpy.tabpy_server.common.util import format_exception +import requests +from tornado import gen +from datetime import timedelta + + +class RestrictedTabPy: + def __init__(self, protocol, port, logger, timeout): + self.protocol = protocol + self.port = port + self.logger = logger + self.timeout = timeout + + def query(self, name, *args, **kwargs): + url = f"{self.protocol}://localhost:{self.port}/query/{name}" + self.logger.log(logging.DEBUG, f"Querying {url}...") + internal_data = {"data": args or kwargs} + data = json.dumps(internal_data) + headers = {"content-type": "application/json"} + response = requests.post( + url=url, data=data, headers=headers, timeout=self.timeout, verify=False + ) + return response.json() + + +class EvaluationPlaneHandler(BaseHandler): + """ + EvaluationPlaneHandler is responsible for running arbitrary python scripts. + """ + + def initialize(self, executor, app): + super(EvaluationPlaneHandler, self).initialize(app) + self.executor = executor + self._error_message_timeout = ( + f"User defined script timed out. " + f"Timeout is set to {self.eval_timeout} s." + ) + + @gen.coroutine + def _post_impl(self): + body = json.loads(self.request.body.decode("utf-8")) + self.logger.log(logging.DEBUG, f"Processing POST request '{body}'...") + if "script" not in body: + self.error_out(400, "Script is empty.") + return + + # Transforming user script into a proper function. + user_code = body["script"] + arguments = None + arguments_str = "" + if "data" in body: + arguments = body["data"] + + if arguments is not None: + if not isinstance(arguments, dict): + self.error_out( + 400, "Script parameters need to be provided as a dictionary." + ) + return + args_in = sorted(arguments.keys()) + n = len(arguments) + if sorted('_arg'+str(i+1) for i in range(n)) == args_in: + arguments_str = ", " + ", ".join(args_in) + else: + self.error_out( + 400, + "Variables names should follow " + "the format _arg1, _arg2, _argN", + ) + return + + function_to_evaluate = f"def _user_script(tabpy{arguments_str}):\n" + for u in user_code.splitlines(): + function_to_evaluate += " " + u + "\n" + + self.logger.log( + logging.INFO, f"function to evaluate={function_to_evaluate}" + ) + + try: + result = yield self._call_subprocess(function_to_evaluate, arguments) + except ( + gen.TimeoutError, + requests.exceptions.ConnectTimeout, + requests.exceptions.ReadTimeout, + ): + self.logger.log(logging.ERROR, self._error_message_timeout) + self.error_out(408, self._error_message_timeout) + return + + if result is not None: + self.write(simplejson.dumps(result, ignore_nan=True)) + else: + self.write("null") + self.finish() + + @gen.coroutine + def post(self): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + self._add_CORS_header() + try: + yield self._post_impl() + except Exception as e: + err_msg = f"{e.__class__.__name__} : {str(e)}" + if err_msg != "KeyError : 'response'": + err_msg = format_exception(e, "POST /evaluate") + self.error_out(500, "Error processing script", info=err_msg) + else: + self.error_out( + 404, + "Error processing script", + info="The endpoint you're " + "trying to query did not respond. Please make sure the " + "endpoint exists and the correct set of arguments are " + "provided.", + ) + + @gen.coroutine + def _call_subprocess(self, function_to_evaluate, arguments): + restricted_tabpy = RestrictedTabPy( + self.protocol, self.port, self.logger, self.eval_timeout + ) + # Exec does not run the function, so it does not block. + exec(function_to_evaluate, globals()) + + # 'noqa' comments below tell flake8 to ignore undefined _user_script + # name - the name is actually defined with user script being wrapped + # in _user_script function (constructed as a striong) and then executed + # with exec() call above. + future = self.executor.submit(_user_script, # noqa: F821 + restricted_tabpy, + **arguments if arguments is not None else None) + + ret = yield gen.with_timeout(timedelta(seconds=self.eval_timeout), future) + raise gen.Return(ret) diff --git a/tabpy/tabpy_server/handlers/main_handler.py b/tabpy/tabpy_server/handlers/main_handler.py new file mode 100644 index 00000000..dbf2680b --- /dev/null +++ b/tabpy/tabpy_server/handlers/main_handler.py @@ -0,0 +1,7 @@ +from tabpy.tabpy_server.handlers import BaseHandler + + +class MainHandler(BaseHandler): + def get(self): + self._add_CORS_header() + self.render("/static/index.html") diff --git a/tabpy/tabpy_server/handlers/management_handler.py b/tabpy/tabpy_server/handlers/management_handler.py new file mode 100644 index 00000000..731dc630 --- /dev/null +++ b/tabpy/tabpy_server/handlers/management_handler.py @@ -0,0 +1,150 @@ +import logging +import os +import shutil +from re import compile as _compile +from uuid import uuid4 as random_uuid + +from tornado import gen + +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers import MainHandler +from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD +from tabpy.tabpy_server.management.state import get_query_object_path +from tabpy.tabpy_server.psws.callbacks import on_state_change + + +def copy_from_local(localpath, remotepath, is_dir=False): + if is_dir: + if not os.path.exists(remotepath): + # remote folder does not exist + shutil.copytree(localpath, remotepath) + else: + # remote folder exists, copy each file + src_files = os.listdir(localpath) + for file_name in src_files: + full_file_name = os.path.join(localpath, file_name) + if os.path.isdir(full_file_name): + # copy folder recursively + full_remote_path = os.path.join(remotepath, file_name) + shutil.copytree(full_file_name, full_remote_path) + else: + # copy each file + shutil.copy(full_file_name, remotepath) + else: + shutil.copy(localpath, remotepath) + + +class ManagementHandler(MainHandler): + def initialize(self, app): + super(ManagementHandler, self).initialize(app) + self.port = self.settings[SettingsParameters.Port] + + def _get_protocol(self): + return "http://" + + @gen.coroutine + def _add_or_update_endpoint(self, action, name, version, request_data): + """ + Add or update an endpoint + """ + self.logger.log(logging.DEBUG, f"Adding/updating model {name}...") + + if not isinstance(name, str): + msg = "Endpoint name must be a string" + self.logger.log(logging.CRITICAL, msg) + raise TypeError(msg) + + name_checker = _compile(r"^[a-zA-Z0-9-_\s]+$") + if not name_checker.match(name): + raise gen.Return( + "endpoint name can only contain: a-z, A-Z, 0-9," + " underscore, hyphens and spaces." + ) + + if self.settings.get("add_or_updating_endpoint"): + msg = ( + "Another endpoint update is already in progress" + ", please wait a while and try again" + ) + self.logger.log(logging.CRITICAL, msg) + raise RuntimeError(msg) + + self.settings["add_or_updating_endpoint"] = random_uuid() + try: + docstring = None + if "docstring" in request_data: + docstring = str( + bytes(request_data["docstring"], "utf-8").decode("unicode_escape") + ) + + description = request_data.get("description", None) + endpoint_type = request_data.get("type", None) + methods = request_data.get("methods", []) + dependencies = request_data.get("dependencies", None) + target = request_data.get("target", None) + schema = request_data.get("schema", None) + src_path = request_data.get("src_path", None) + target_path = get_query_object_path( + self.settings[SettingsParameters.StateFilePath], name, version + ) + + path_checker = _compile(r"^[\\\:a-zA-Z0-9-_~\s/\.\(\)]+$") + # copy from staging + if src_path: + if not isinstance(src_path, str): + raise gen.Return("src_path must be a string.") + if not path_checker.match(src_path): + raise gen.Return(f"Invalid source path for endpoint {name}") + + yield self._copy_po_future(src_path, target_path) + elif endpoint_type != "alias": + raise gen.Return("src_path is required to add/update an endpoint.") + else: + # alias special logic: + if not target: + raise gen.Return("Target is required for alias endpoint.") + dependencies = [target] + + # update local config + try: + if action == "add": + self.tabpy_state.add_endpoint( + name=name, + description=description, + docstring=docstring, + endpoint_type=endpoint_type, + methods=methods, + dependencies=dependencies, + target=target, + schema=schema, + ) + else: + self.tabpy_state.update_endpoint( + name=name, + description=description, + docstring=docstring, + endpoint_type=endpoint_type, + methods=methods, + dependencies=dependencies, + target=target, + schema=schema, + version=version, + ) + + except Exception as e: + raise gen.Return(f"Error when changing TabPy state: {e}") + + on_state_change( + self.settings, self.tabpy_state, self.python_service, self.logger + ) + + finally: + self.settings["add_or_updating_endpoint"] = None + + @gen.coroutine + def _copy_po_future(self, src_path, target_path): + future = STAGING_THREAD.submit( + copy_from_local, src_path, target_path, is_dir=True + ) + ret = yield future + raise gen.Return(ret) diff --git a/tabpy/tabpy_server/handlers/query_plane_handler.py b/tabpy/tabpy_server/handlers/query_plane_handler.py new file mode 100644 index 00000000..aab42593 --- /dev/null +++ b/tabpy/tabpy_server/handlers/query_plane_handler.py @@ -0,0 +1,233 @@ +from tabpy.tabpy_server.handlers import BaseHandler +import logging +import time +from tabpy.tabpy_server.common.messages import ( + Query, + QuerySuccessful, + QueryError, + UnknownURI, +) +from hashlib import md5 +import uuid +import json +from tabpy.tabpy_server.common.util import format_exception +import urllib +from tornado import gen + + +def _get_uuid(): + """Generate a unique identifier string""" + return str(uuid.uuid4()) + + +class QueryPlaneHandler(BaseHandler): + def initialize(self, app): + super(QueryPlaneHandler, self).initialize(app) + + def _query(self, po_name, data, uid, qry): + """ + Parameters + ---------- + po_name : str + The name of the query object to query + + data : dict + The deserialized request body + + uid: str + A unique identifier for the request + + qry: str + The incoming query object. This object maintains + raw incoming request, which is different from the sanitied data + + Returns + ------- + out : (result type, dict, int) + A triple containing a result type, the result message + as a dictionary, and the time in seconds that it took to complete + the request. + """ + self.logger.log(logging.DEBUG, f"Collecting query info for {po_name}...") + start_time = time.time() + response = self.python_service.ps.query(po_name, data, uid) + gls_time = time.time() - start_time + self.logger.log(logging.DEBUG, f"Query info: {response}") + + if isinstance(response, QuerySuccessful): + response_json = response.to_json() + md5_tag = md5(response_json.encode("utf-8")).hexdigest() + self.set_header("Etag", f'"{md5_tag}"') + return (QuerySuccessful, response.for_json(), gls_time) + else: + self.logger.log(logging.ERROR, f"Failed query, response: {response}") + return (type(response), response.for_json(), gls_time) + + # handle HTTP Options requests to support CORS + # don't check API key (client does not send or receive data for OPTIONS, + # it just allows the client to subsequently make a POST request) + def options(self, pred_name): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + self.logger.log(logging.DEBUG, f"Processing OPTIONS for /query/{pred_name}") + + # add CORS headers if TabPy has a cors_origin specified + self._add_CORS_header() + self.write({}) + + def _handle_result(self, po_name, data, qry, uid): + (response_type, response, gls_time) = self._query(po_name, data, uid, qry) + + if response_type == QuerySuccessful: + result_dict = { + "response": response["response"], + "version": response["version"], + "model": po_name, + "uuid": uid, + } + self.write(result_dict) + self.finish() + return (gls_time, response["response"]) + else: + if response_type == UnknownURI: + self.error_out( + 404, + "UnknownURI", + info=( + "No query object has been registered" + f' with the name "{po_name}"' + ), + ) + elif response_type == QueryError: + self.error_out(400, "QueryError", info=response) + else: + self.error_out(500, "Error querying GLS", info=response) + + return (None, None) + + def _sanitize_request_data(self, data): + if not isinstance(data, dict): + msg = "Input data must be a dictionary" + self.logger.log(logging.CRITICAL, msg) + raise RuntimeError(msg) + + if "method" in data: + return {"data": data.get("data"), "method": data.get("method")} + elif "data" in data: + return data.get("data") + else: + msg = 'Input data must be a dictionary with a key called "data"' + self.logger.log(logging.CRITICAL, msg) + raise RuntimeError(msg) + + def _process_query(self, endpoint_name, start): + self.logger.log(logging.DEBUG, f"Processing query {endpoint_name}...") + try: + self._add_CORS_header() + + if not self.request.body: + self.request.body = {} + + # extract request data explicitly for caching purpose + request_json = self.request.body.decode("utf-8") + + # Sanitize input data + data = self._sanitize_request_data(json.loads(request_json)) + except Exception as e: + self.logger.log(logging.ERROR, str(e)) + err_msg = format_exception(e, "Invalid Input Data") + self.error_out(400, err_msg) + return + + try: + (po_name, _) = self._get_actual_model(endpoint_name) + + # po_name is None if self.python_service.ps.query_objects.get( + # endpoint_name) is None + if not po_name: + self.error_out( + 404, "UnknownURI", info=f'Endpoint "{endpoint_name}" does not exist' + ) + return + + po_obj = self.python_service.ps.query_objects.get(po_name) + + if not po_obj: + self.error_out( + 404, "UnknownURI", info=f'Endpoint "{po_name}" does not exist' + ) + return + + if po_name != endpoint_name: + self.logger.log( + logging.INFO, f"Querying actual model: po_name={po_name}" + ) + + uid = _get_uuid() + + # record query w/ request ID in query log + qry = Query(po_name, request_json) + gls_time = 0 + # send a query to PythonService and return + (gls_time, _) = self._handle_result(po_name, data, qry, uid) + + # if error occurred, GLS time is None. + if not gls_time: + return + + except Exception as e: + self.logger.log(logging.ERROR, str(e)) + err_msg = format_exception(e, "process query") + self.error_out(500, "Error processing query", info=err_msg) + return + + def _get_actual_model(self, endpoint_name): + # Find the actual query to run from given endpoint + all_endpoint_names = [] + + while True: + endpoint_info = self.python_service.ps.query_objects.get(endpoint_name) + if not endpoint_info: + return [None, None] + + all_endpoint_names.append(endpoint_name) + + endpoint_type = endpoint_info.get("type", "model") + + if endpoint_type == "alias": + endpoint_name = endpoint_info["endpoint_obj"] + elif endpoint_type == "model": + break + else: + self.error_out( + 500, + "Unknown endpoint type", + info=f'Endpoint type "{endpoint_type}" does not exist', + ) + return + + return (endpoint_name, all_endpoint_names) + + @gen.coroutine + def get(self, endpoint_name): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + start = time.time() + endpoint_name = urllib.parse.unquote(endpoint_name) + self._process_query(endpoint_name, start) + + @gen.coroutine + def post(self, endpoint_name): + self.logger.log(logging.DEBUG, f"Processing POST for /query/{endpoint_name}...") + + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + start = time.time() + endpoint_name = urllib.parse.unquote(endpoint_name) + self._process_query(endpoint_name, start) diff --git a/tabpy/tabpy_server/handlers/service_info_handler.py b/tabpy/tabpy_server/handlers/service_info_handler.py new file mode 100644 index 00000000..51152a98 --- /dev/null +++ b/tabpy/tabpy_server/handlers/service_info_handler.py @@ -0,0 +1,23 @@ +import json +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers import ManagementHandler + + +class ServiceInfoHandler(ManagementHandler): + def initialize(self, app): + super(ServiceInfoHandler, self).initialize(app) + + def get(self): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + self._add_CORS_header() + info = {} + info["description"] = self.tabpy_state.get_description() + info["creation_time"] = self.tabpy_state.creation_time + info["state_path"] = self.settings[SettingsParameters.StateFilePath] + info["server_version"] = self.settings[SettingsParameters.ServerVersion] + info["name"] = self.tabpy_state.name + info["versions"] = self.settings[SettingsParameters.ApiVersions] + self.write(json.dumps(info)) diff --git a/tabpy/tabpy_server/handlers/status_handler.py b/tabpy/tabpy_server/handlers/status_handler.py new file mode 100644 index 00000000..2f743b3f --- /dev/null +++ b/tabpy/tabpy_server/handlers/status_handler.py @@ -0,0 +1,29 @@ +import json +import logging +from tabpy.tabpy_server.handlers import BaseHandler + + +class StatusHandler(BaseHandler): + def initialize(self, app): + super(StatusHandler, self).initialize(app) + + def get(self): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + self._add_CORS_header() + + status_dict = {} + for k, v in self.python_service.ps.query_objects.items(): + status_dict[k] = { + "version": v["version"], + "type": v["type"], + "status": v["status"], + "last_error": v["last_error"], + } + + self.logger.log(logging.DEBUG, f"Found models: {status_dict}") + self.write(json.dumps(status_dict)) + self.finish() + return diff --git a/tabpy/tabpy_server/handlers/upload_destination_handler.py b/tabpy/tabpy_server/handlers/upload_destination_handler.py new file mode 100644 index 00000000..729aff3e --- /dev/null +++ b/tabpy/tabpy_server/handlers/upload_destination_handler.py @@ -0,0 +1,20 @@ +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.handlers import ManagementHandler +import os + + +_QUERY_OBJECT_STAGING_FOLDER = "staging" + + +class UploadDestinationHandler(ManagementHandler): + def initialize(self, app): + super(UploadDestinationHandler, self).initialize(app) + + def get(self): + if self.should_fail_with_not_authorized(): + self.fail_with_not_authorized() + return + + path = self.settings[SettingsParameters.StateFilePath] + path = os.path.join(path, _QUERY_OBJECT_STAGING_FOLDER) + self.write({"path": path}) diff --git a/tabpy/tabpy_server/handlers/util.py b/tabpy/tabpy_server/handlers/util.py new file mode 100644 index 00000000..14e029c6 --- /dev/null +++ b/tabpy/tabpy_server/handlers/util.py @@ -0,0 +1,32 @@ +import binascii +from hashlib import pbkdf2_hmac + + +def hash_password(username, pwd): + """ + Hashes password using PKDBF2 method: + hash = PKDBF2('sha512', pwd, salt=username, 10000) + + Parameters + ---------- + username : str + User name (login). Used as salt for hashing. + User name is lowercased befor being used in hashing. + Salt is formatted as '_$salt@tabpy:$_' to + guarantee there's at least 16 characters. + + pwd : str + Password to hash. + + Returns + ------- + str + Sting representation (hexidecimal) for PBKDF2 hash + for the password. + """ + salt = f"_$salt@tabpy:{username.lower()}$_" + + hash = pbkdf2_hmac( + hash_name="sha512", password=pwd.encode(), salt=salt.encode(), iterations=10000 + ) + return binascii.hexlify(hash).decode() diff --git a/tabpy/tabpy_server/management/__init__.py b/tabpy/tabpy_server/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy/tabpy_server/management/state.py b/tabpy/tabpy_server/management/state.py new file mode 100644 index 00000000..424d3b16 --- /dev/null +++ b/tabpy/tabpy_server/management/state.py @@ -0,0 +1,643 @@ +try: + from ConfigParser import ConfigParser +except ImportError: + from configparser import ConfigParser +import json +import logging +from tabpy.tabpy_server.management.util import write_state_config +from threading import Lock +from time import time + + +logger = logging.getLogger(__name__) + +# State File Config Section Names +_DEPLOYMENT_SECTION_NAME = "Query Objects Service Versions" +_QUERY_OBJECT_DOCSTRING = "Query Objects Docstrings" +_SERVICE_INFO_SECTION_NAME = "Service Info" +_META_SECTION_NAME = "Meta" + +# Directory Names +_QUERY_OBJECT_DIR = "query_objects" + +""" +Lock to change the TabPy State. +""" +_PS_STATE_LOCK = Lock() + + +def state_lock(func): + """ + Mutex for changing PS state + """ + + def wrapper(self, *args, **kwargs): + try: + _PS_STATE_LOCK.acquire() + return func(self, *args, **kwargs) + finally: + # ALWAYS RELEASE LOCK + _PS_STATE_LOCK.release() + + return wrapper + + +def _get_root_path(state_path): + if state_path[-1] != "/": + state_path += "/" + + return state_path + + +def get_query_object_path(state_file_path, name, version): + """ + Returns the query object path + + If the version is None, a path without the version will be returned. + """ + root_path = _get_root_path(state_file_path) + sub_path = [_QUERY_OBJECT_DIR, name] + if version is not None: + sub_path.append(str(version)) + full_path = root_path + "/".join(sub_path) + return full_path + + +class TabPyState: + """ + The TabPy state object that stores attributes + about this TabPy and perform GET/SET on these + attributes. + + Attributes: + - name + - description + - endpoints (name, description, docstring, version, target) + - revision number + + When the state object is initialized, the state is saved as a ConfigParser. + There is a config to any attribute. + + """ + + def __init__(self, settings, config=None): + self.settings = settings + self.set_config(config, _update=False) + + @state_lock + def set_config(self, config, logger=logging.getLogger(__name__), _update=True): + """ + Set the local ConfigParser manually. + This new ConfigParser will be used as current state. + """ + if not isinstance(config, ConfigParser): + raise ValueError("Invalid config") + self.config = config + if _update: + self._write_state(logger) + + def get_endpoints(self, name=None): + """ + Return a dictionary of endpoints + + Parameters + ---------- + name : str + The name of the endpoint. + If "name" is specified, only the information about that endpoint + will be returned. + + Returns + ------- + endpoints : dict + The dictionary containing information about each endpoint. + The keys are the endpoint names. + The values for each include: + - description + - doc string + - type + - target + + """ + endpoints = {} + try: + endpoint_names = self._get_config_value(_DEPLOYMENT_SECTION_NAME, name) + except Exception as e: + logger.error(f"error in get_endpoints: {str(e)}") + return {} + + if name: + endpoint_info = json.loads(endpoint_names) + docstring = self._get_config_value(_QUERY_OBJECT_DOCSTRING, name) + endpoint_info["docstring"] = str( + bytes(docstring, "utf-8").decode("unicode_escape") + ) + endpoints = {name: endpoint_info} + else: + for endpoint_name in endpoint_names: + endpoint_info = json.loads( + self._get_config_value(_DEPLOYMENT_SECTION_NAME, endpoint_name) + ) + docstring = self._get_config_value( + _QUERY_OBJECT_DOCSTRING, endpoint_name, True, "" + ) + endpoint_info["docstring"] = str( + bytes(docstring, "utf-8").decode("unicode_escape") + ) + endpoints[endpoint_name] = endpoint_info + logger.debug(f"Collected endpoints: {endpoints}") + return endpoints + + def _check_endpoint_exists(self, name): + endpoints = self.get_endpoints() + if not name or not isinstance(name, str) or len(name) == 0: + raise ValueError("name of the endpoint must be a valid string.") + + return name in endpoints + + def _check_and_set_endpoint_str_value(self, param, paramName, defaultValue): + if not param and defaultValue is not None: + return defaultValue + + if not param or not isinstance(param, str): + raise ValueError(f"{paramName} must be a string.") + + return param + + def _check_and_set_endpoint_description(self, description, defaultValue): + return self._check_and_set_endpoint_str_value(description, "description", defaultValue) + + def _check_and_set_endpoint_docstring(self, docstring, defaultValue): + return self._check_and_set_endpoint_str_value(docstring, "docstring", defaultValue) + + def _check_and_set_endpoint_type(self, endpoint_type, defaultValue): + return self._check_and_set_endpoint_str_value( + endpoint_type, "endpoint type", defaultValue) + + def _check_and_set_target(self, target, defaultValue): + return self._check_and_set_endpoint_str_value( + target, "target", defaultValue) + + def _check_and_set_dependencies(self, dependencies, defaultValue): + if not dependencies: + return defaultValue + + if dependencies or not isinstance(dependencies, list): + raise ValueError("dependencies must be a list.") + + return dependencies + + @state_lock + def add_endpoint( + self, + name, + description=None, + docstring=None, + endpoint_type=None, + methods=None, + target=None, + dependencies=None, + schema=None, + ): + """ + Add a new endpoint to the TabPy. + + Parameters + ---------- + name : str + Name of the endpoint + description : str, optional + Description of this endpoint + doc_string : str, optional + The doc string for this endpoint, if needed. + endpoint_type : str + The endpoint type (model, alias) + target : str, optional + The target endpoint name for the alias to be added. + + Note: + The version of this endpoint will be set to 1 since it is a new + endpoint. + + """ + try: + if (self._check_endpoint_exists(name)): + raise ValueError(f"endpoint {name} already exists.") + + endpoints = self.get_endpoints() + + description = self._check_and_set_endpoint_description(description, "") + docstring = self._check_and_set_endpoint_docstring( + docstring, "-- no docstring found in query function --") + endpoint_type = self._check_and_set_endpoint_type(endpoint_type, None) + dependencies = self._check_and_set_dependencies(dependencies, []) + + target = self._check_and_set_target(target, "") + if target and target not in endpoints: + raise ValueError("target endpoint is not valid.") + + endpoint_info = { + "description": description, + "docstring": docstring, + "type": endpoint_type, + "version": 1, + "dependencies": dependencies, + "target": target, + "creation_time": int(time()), + "last_modified_time": int(time()), + "schema": schema, + } + + endpoints[name] = endpoint_info + self._add_update_endpoints_config(endpoints) + except Exception as e: + logger.error(f"Error in add_endpoint: {e}") + raise + + def _add_update_endpoints_config(self, endpoints): + # save the endpoint info to config + dstring = "" + for endpoint_name in endpoints: + try: + info = endpoints[endpoint_name] + dstring = str( + bytes(info["docstring"], "utf-8").decode("unicode_escape") + ) + self._set_config_value( + _QUERY_OBJECT_DOCSTRING, + endpoint_name, + dstring, + _update_revision=False, + ) + del info["docstring"] + self._set_config_value( + _DEPLOYMENT_SECTION_NAME, endpoint_name, json.dumps(info) + ) + except Exception as e: + logger.error(f"Unable to write endpoints config: {e}") + raise + + @state_lock + def update_endpoint( + self, + name, + description=None, + docstring=None, + endpoint_type=None, + version=None, + methods=None, + target=None, + dependencies=None, + schema=None, + ): + """ + Update an existing endpoint on the TabPy. + + Parameters + ---------- + name : str + Name of the endpoint + description : str, optional + Description of this endpoint + doc_string : str, optional + The doc string for this endpoint, if needed. + endpoint_type : str, optional + The endpoint type (model, alias) + version : str, optional + The version of this endpoint + dependencies=[] + List of dependent endpoints for this existing endpoint + target : str, optional + The target endpoint name for the alias. + + Note: + For those parameters that are not specified, those values will not + get changed. + + """ + try: + if (not self._check_endpoint_exists(name)): + raise ValueError(f"endpoint {name} does not exist.") + + endpoints = self.get_endpoints() + endpoint_info = endpoints[name] + + description = self._check_and_set_endpoint_description( + description, endpoint_info["description"]) + docstring = self._check_and_set_endpoint_docstring( + docstring, endpoint_info["docstring"]) + endpoint_type = self._check_and_set_endpoint_type( + endpoint_type, endpoint_info["type"]) + dependencies = self._check_and_set_dependencies( + dependencies, endpoint_info.get("dependencies", [])) + + target = self._check_and_set_target(target, None) + if target and target not in endpoints: + raise ValueError("target endpoint is not valid.") + elif not target: + target = endpoint_info["target"] + + if version and not isinstance(version, int): + raise ValueError("version must be an int.") + elif not version: + version = endpoint_info["version"] + + endpoint_info = { + "description": description, + "docstring": docstring, + "type": endpoint_type, + "version": version, + "dependencies": dependencies, + "target": target, + "creation_time": endpoint_info["creation_time"], + "last_modified_time": int(time()), + "schema": schema, + } + + endpoints[name] = endpoint_info + self._add_update_endpoints_config(endpoints) + except Exception as e: + logger.error(f"Error in update_endpoint: {e}") + raise + + @state_lock + def delete_endpoint(self, name): + """ + Delete an existing endpoint on the TabPy + + Parameters + ---------- + name : str + The name of the endpoint to be deleted. + + Returns + ------- + deleted endpoint object + + Note: + Cannot delete this endpoint if other endpoints are currently + depending on this endpoint. + + """ + if not name or name == "": + raise ValueError("Name of the endpoint must be a valid string.") + endpoints = self.get_endpoints() + if name not in endpoints: + raise ValueError(f"Endpoint {name} does not exist.") + + endpoint_to_delete = endpoints[name] + + # get dependencies and target + deps = set() + for endpoint_name in endpoints: + if endpoint_name != name: + deps_list = endpoints[endpoint_name].get("dependencies", []) + if name in deps_list: + deps.add(endpoint_name) + + # check if other endpoints are depending on this endpoint + if len(deps) > 0: + raise ValueError( + f"Cannot remove endpoint {name}, it is currently " + f"used by {list(deps)} endpoints." + ) + + del endpoints[name] + + # delete the endpoint from state + try: + self._remove_config_option( + _QUERY_OBJECT_DOCSTRING, name, _update_revision=False + ) + self._remove_config_option(_DEPLOYMENT_SECTION_NAME, name) + + return endpoint_to_delete + except Exception as e: + logger.error(f"Unable to delete endpoint {e}") + raise ValueError(f"Unable to delete endpoint: {e}") + + @property + def name(self): + """ + Returns the name of the TabPy service. + """ + name = None + try: + name = self._get_config_value(_SERVICE_INFO_SECTION_NAME, "Name") + except Exception as e: + logger.error(f"Unable to get name: {e}") + return name + + @property + def creation_time(self): + """ + Returns the creation time of the TabPy service. + """ + creation_time = 0 + try: + creation_time = self._get_config_value( + _SERVICE_INFO_SECTION_NAME, "Creation Time" + ) + except Exception as e: + logger.error(f"Unable to get name: {e}") + return creation_time + + @state_lock + def set_name(self, name): + """ + Set the name of this TabPy service. + + Parameters + ---------- + name : str + Name of TabPy service. + """ + if not isinstance(name, str): + raise ValueError("name must be a string.") + try: + self._set_config_value(_SERVICE_INFO_SECTION_NAME, "Name", name) + except Exception as e: + logger.error(f"Unable to set name: {e}") + + def get_description(self): + """ + Returns the description of the TabPy service. + """ + description = None + try: + description = self._get_config_value( + _SERVICE_INFO_SECTION_NAME, "Description" + ) + except Exception as e: + logger.error(f"Unable to get description: {e}") + return description + + @state_lock + def set_description(self, description): + """ + Set the description of this TabPy service. + + Parameters + ---------- + description : str + Description of TabPy service. + """ + if not isinstance(description, str): + raise ValueError("Description must be a string.") + try: + self._set_config_value( + _SERVICE_INFO_SECTION_NAME, "Description", description + ) + except Exception as e: + logger.error(f"Unable to set description: {e}") + + def get_revision_number(self): + """ + Returns the revision number of this TabPy service. + """ + rev = -1 + try: + rev = int(self._get_config_value(_META_SECTION_NAME, "Revision Number")) + except Exception as e: + logger.error(f"Unable to get revision number: {e}") + return rev + + def get_access_control_allow_origin(self): + """ + Returns Access-Control-Allow-Origin of this TabPy service. + """ + _cors_origin = "" + try: + logger.debug("Collecting Access-Control-Allow-Origin from state file ...") + _cors_origin = self._get_config_value( + "Service Info", "Access-Control-Allow-Origin" + ) + except Exception as e: + logger.error(e) + return _cors_origin + + def get_access_control_allow_headers(self): + """ + Returns Access-Control-Allow-Headers of this TabPy service. + """ + _cors_headers = "" + try: + _cors_headers = self._get_config_value( + "Service Info", "Access-Control-Allow-Headers" + ) + except Exception: + pass + return _cors_headers + + def get_access_control_allow_methods(self): + """ + Returns Access-Control-Allow-Methods of this TabPy service. + """ + _cors_methods = "" + try: + _cors_methods = self._get_config_value( + "Service Info", "Access-Control-Allow-Methods" + ) + except Exception: + pass + return _cors_methods + + def _set_revision_number(self, revision_number): + """ + Set the revision number of this TabPy service. + """ + if not isinstance(revision_number, int): + raise ValueError("revision number must be an int.") + try: + self._set_config_value( + _META_SECTION_NAME, "Revision Number", revision_number + ) + except Exception as e: + logger.error(f"Unable to set revision number: {e}") + + def _remove_config_option( + self, + section_name, + option_name, + logger=logging.getLogger(__name__), + _update_revision=True, + ): + if not self.config: + raise ValueError("State configuration not yet loaded.") + self.config.remove_option(section_name, option_name) + # update revision number + if _update_revision: + self._increase_revision_number() + self._write_state(logger=logger) + + def _has_config_value(self, section_name, option_name): + if not self.config: + raise ValueError("State configuration not yet loaded.") + return self.config.has_option(section_name, option_name) + + def _increase_revision_number(self): + if not self.config: + raise ValueError("State configuration not yet loaded.") + cur_rev = int(self.config.get(_META_SECTION_NAME, "Revision Number")) + self.config.set(_META_SECTION_NAME, "Revision Number", str(cur_rev + 1)) + + def _set_config_value( + self, + section_name, + option_name, + option_value, + logger=logging.getLogger(__name__), + _update_revision=True, + ): + if not self.config: + raise ValueError("State configuration not yet loaded.") + + if not self.config.has_section(section_name): + logger.log(logging.DEBUG, f"Adding config section {section_name}") + self.config.add_section(section_name) + + self.config.set(section_name, option_name, option_value) + # update revision number + if _update_revision: + self._increase_revision_number() + self._write_state(logger=logger) + + def _get_config_items(self, section_name): + if not self.config: + raise ValueError("State configuration not yet loaded.") + return self.config.items(section_name) + + def _get_config_value( + self, section_name, option_name, optional=False, default_value=None + ): + logger.log( + logging.DEBUG, + f"Loading option '{option_name}' from section [{section_name}]...") + + if not self.config: + msg = "State configuration not yet loaded." + logging.log(msg) + raise ValueError(msg) + + res = None + if not option_name: + res = self.config.options(section_name) + elif self.config.has_option(section_name, option_name): + res = self.config.get(section_name, option_name) + elif optional: + res = default_value + else: + raise ValueError( + f"Cannot find option name {option_name} " + f"under section {section_name}" + ) + + logger.log(logging.DEBUG, f"Returning value '{res}'") + return res + + def _write_state(self, logger=logging.getLogger(__name__)): + """ + Write state (ConfigParser) to Consul + """ + logger.log(logging.INFO, "Writing state to config") + write_state_config(self.config, self.settings, logger=logger) diff --git a/tabpy/tabpy_server/management/util.py b/tabpy/tabpy_server/management/util.py new file mode 100644 index 00000000..cb9b7709 --- /dev/null +++ b/tabpy/tabpy_server/management/util.py @@ -0,0 +1,46 @@ +import logging +import os + +try: + from ConfigParser import ConfigParser as _ConfigParser +except ImportError: + from configparser import ConfigParser as _ConfigParser +from tabpy.tabpy_server.app.ConfigParameters import ConfigParameters +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters + + +def write_state_config(state, settings, logger=logging.getLogger(__name__)): + if SettingsParameters.StateFilePath in settings: + state_path = settings[SettingsParameters.StateFilePath] + else: + msg = f"{ConfigParameters.TABPY_STATE_PATH} is not set" + logger.log(logging.CRITICAL, msg) + raise ValueError(msg) + + logger.log(logging.DEBUG, f"State path is {state_path}") + state_key = os.path.join(state_path, "state.ini") + tmp_state_file = state_key + + with open(tmp_state_file, "w") as f: + state.write(f) + + +def _get_state_from_file(state_path, logger=logging.getLogger(__name__)): + state_key = os.path.join(state_path, "state.ini") + tmp_state_file = state_key + + if not os.path.exists(tmp_state_file): + msg = f"Missing config file at {tmp_state_file}" + logger.log(logging.CRITICAL, msg) + raise ValueError(msg) + + config = _ConfigParser(allow_no_value=True) + config.optionxform = str + config.read(tmp_state_file) + + if not config.has_section("Service Info"): + msg = "Config error: Expected [Service Info] section in " f"{tmp_state_file}" + logger.log(logging.CRITICAL, msg) + raise ValueError(msg) + + return config diff --git a/tabpy/tabpy_server/psws/__init__.py b/tabpy/tabpy_server/psws/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy/tabpy_server/psws/callbacks.py b/tabpy/tabpy_server/psws/callbacks.py new file mode 100644 index 00000000..4b1fe14e --- /dev/null +++ b/tabpy/tabpy_server/psws/callbacks.py @@ -0,0 +1,205 @@ +import logging +from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters +from tabpy.tabpy_server.common.messages import ( + LoadObject, + DeleteObjects, + ListObjects, + ObjectList, +) +from tabpy.tabpy_server.common.endpoint_file_mgr import cleanup_endpoint_files +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.management.state import TabPyState, get_query_object_path +from tabpy.tabpy_server.management import util +from time import sleep +from tornado import gen + + +logger = logging.getLogger(__name__) + + +def wait_for_endpoint_loaded(python_service, object_uri): + """ + This method waits for the object to be loaded. + """ + logger.info("Waiting for object to be loaded...") + while True: + msg = ListObjects() + list_object_msg = python_service.manage_request(msg) + if not isinstance(list_object_msg, ObjectList): + logger.error(f"Error loading endpoint {object_uri}: {list_object_msg}") + return + + for (uri, info) in list_object_msg.objects.items(): + if uri == object_uri: + if info["status"] != "LoadInProgress": + logger.info(f'Object load status: {info["status"]}') + return + + sleep(0.1) + + +@gen.coroutine +def init_ps_server(settings, tabpy_state): + logger.info("Initializing TabPy Server...") + existing_pos = tabpy_state.get_endpoints() + for (object_name, obj_info) in existing_pos.items(): + try: + object_version = obj_info["version"] + get_query_object_path( + settings[SettingsParameters.StateFilePath], object_name, object_version + ) + except Exception as e: + logger.error( + f"Exception encounted when downloading object: {object_name}" + f", error: {e}" + ) + + +@gen.coroutine +def init_model_evaluator(settings, tabpy_state, python_service): + """ + This will go through all models that the service currently have and + initialize them. + """ + logger.info("Initializing models...") + + existing_pos = tabpy_state.get_endpoints() + + for (object_name, obj_info) in existing_pos.items(): + object_version = obj_info["version"] + object_type = obj_info["type"] + object_path = get_query_object_path( + settings[SettingsParameters.StateFilePath], object_name, object_version + ) + + logger.info( + f"Load endpoint: {object_name}, " + f"version: {object_version}, " + f"type: {object_type}" + ) + if object_type == "alias": + msg = LoadObject( + object_name, obj_info["target"], object_version, False, "alias" + ) + else: + local_path = object_path + msg = LoadObject( + object_name, local_path, object_version, False, object_type + ) + python_service.manage_request(msg) + + +def _get_latest_service_state(settings, tabpy_state, new_ps_state, python_service): + """ + Update the endpoints from the latest remote state file. + + Returns + -------- + (has_changes, endpoint_diff): + has_changes: True or False + endpoint_diff: Summary of what has changed, one entry for each changes + """ + # Shortcut when nothing is changed + changes = {"endpoints": {}} + + # update endpoints + new_endpoints = new_ps_state.get_endpoints() + diff = {} + current_endpoints = python_service.ps.query_objects + for (endpoint_name, endpoint_info) in new_endpoints.items(): + existing_endpoint = current_endpoints.get(endpoint_name) + if (existing_endpoint is None) or endpoint_info["version"] != existing_endpoint[ + "version" + ]: + # Either a new endpoint or new endpoint version + path_to_new_version = get_query_object_path( + settings[SettingsParameters.StateFilePath], + endpoint_name, + endpoint_info["version"], + ) + endpoint_type = endpoint_info.get("type", "model") + diff[endpoint_name] = ( + endpoint_type, + endpoint_info["version"], + path_to_new_version, + ) + + # add removed models too + for (endpoint_name, endpoint_info) in current_endpoints.items(): + if endpoint_name not in new_endpoints.keys(): + endpoint_type = current_endpoints[endpoint_name].get("type", "model") + diff[endpoint_name] = (endpoint_type, None, None) + + if diff: + changes["endpoints"] = diff + + return (True, changes) + + +@gen.coroutine +def on_state_change( + settings, tabpy_state, python_service, logger=logging.getLogger(__name__) +): + try: + logger.log(logging.INFO, "Loading state from state file") + config = util._get_state_from_file( + settings[SettingsParameters.StateFilePath], logger=logger + ) + new_ps_state = TabPyState(config=config, settings=settings) + + (has_changes, changes) = _get_latest_service_state( + settings, tabpy_state, new_ps_state, python_service + ) + if not has_changes: + logger.info("Nothing changed, return.") + return + + new_endpoints = new_ps_state.get_endpoints() + for object_name in changes["endpoints"]: + (object_type, object_version, object_path) = changes["endpoints"][ + object_name + ] + + if not object_path and not object_version: # removal + logger.info(f"Removing object: URI={object_name}") + + python_service.manage_request(DeleteObjects([object_name])) + + cleanup_endpoint_files( + object_name, settings[SettingsParameters.UploadDir], logger=logger + ) + + else: + endpoint_info = new_endpoints[object_name] + is_update = object_version > 1 + if object_type == "alias": + msg = LoadObject( + object_name, + endpoint_info["target"], + object_version, + is_update, + "alias", + ) + else: + local_path = object_path + msg = LoadObject( + object_name, local_path, object_version, is_update, object_type + ) + + python_service.manage_request(msg) + wait_for_endpoint_loaded(python_service, object_name) + + # cleanup old version of endpoint files + if object_version > 2: + cleanup_endpoint_files( + object_name, + settings[SettingsParameters.UploadDir], + logger=logger, + retain_versions=[object_version, object_version - 1], + ) + + except Exception as e: + err_msg = format_exception(e, "on_state_change") + logger.log( + logging.ERROR, f"Error submitting update model request: error={err_msg}" + ) diff --git a/tabpy/tabpy_server/psws/python_service.py b/tabpy/tabpy_server/psws/python_service.py new file mode 100644 index 00000000..07fa6a3a --- /dev/null +++ b/tabpy/tabpy_server/psws/python_service.py @@ -0,0 +1,275 @@ +import concurrent.futures +import logging +from tabpy.tabpy_server.common.util import format_exception +from tabpy.tabpy_server.common.messages import ( + LoadObject, + DeleteObjects, + FlushObjects, + CountObjects, + ListObjects, + UnknownMessage, + LoadFailed, + ObjectsDeleted, + ObjectsFlushed, + QueryFailed, + QuerySuccessful, + UnknownURI, + DownloadSkipped, + LoadInProgress, + ObjectCount, + ObjectList, +) +from tabpy.tabpy_tools.query_object import QueryObject + + +logger = logging.getLogger(__name__) + + +class PythonServiceHandler: + """ + A wrapper around PythonService object that receives requests and calls the + corresponding methods. + """ + + def __init__(self, ps): + self.ps = ps + + def manage_request(self, msg): + try: + logger.debug(f"Received request {type(msg).__name__}") + if isinstance(msg, LoadObject): + response = self.ps.load_object(*msg) + elif isinstance(msg, DeleteObjects): + response = self.ps.delete_objects(msg.uris) + elif isinstance(msg, FlushObjects): + response = self.ps.flush_objects() + elif isinstance(msg, CountObjects): + response = self.ps.count_objects() + elif isinstance(msg, ListObjects): + response = self.ps.list_objects() + else: + response = UnknownMessage(msg) + + logger.debug(f"Returning response {response}") + return response + except Exception as e: + logger.exception(e) + msg = e + if hasattr(e, "message"): + msg = e.message + logger.error(f"Error processing request: {msg}") + return UnknownMessage(msg) + + +class PythonService: + """ + This class is a simple wrapper maintaining loaded query objects from + the current TabPy instance. `query_objects` is a dictionary that + maps query object URI to query objects + + The query_objects schema is as follow: + + {'version': , + 'last_error':, + 'endpoint_obj':, + 'type':, + 'status':} + + """ + + def __init__(self, query_objects=None): + + self.EXECUTOR = concurrent.futures.ThreadPoolExecutor(max_workers=1) + self.query_objects = query_objects or {} + + def _load_object( + self, object_uri, object_url, object_version, is_update, object_type + ): + try: + logger.info( + f"Loading object:, URI={object_uri}, " + f"URL={object_url}, version={object_version}, " + f"is_updated={is_update}" + ) + if object_type == "model": + po = QueryObject.load(object_url) + elif object_type == "alias": + po = object_url + else: + raise RuntimeError(f"Unknown object type: {object_type}") + + self.query_objects[object_uri] = { + "version": object_version, + "type": object_type, + "endpoint_obj": po, + "status": "LoadSuccessful", + "last_error": None, + } + except Exception as e: + logger.exception(e) + logger.error( + f"Unable to load QueryObject: path={object_url}, " f"error={str(e)}" + ) + + self.query_objects[object_uri] = { + "version": object_version, + "type": object_type, + "endpoint_obj": None, + "status": "LoadFailed", + "last_error": f"Load failed: {str(e)}", + } + + def load_object( + self, object_uri, object_url, object_version, is_update, object_type + ): + try: + obj_info = self.query_objects.get(object_uri) + if ( + obj_info + and obj_info["endpoint_obj"] + and (obj_info["version"] >= object_version) + ): + logger.info("Received load message for object already loaded") + + return DownloadSkipped( + object_uri, + obj_info["version"], + "Object with greater " "or equal version already loaded", + ) + else: + if object_uri not in self.query_objects: + self.query_objects[object_uri] = { + "version": object_version, + "type": object_type, + "endpoint_obj": None, + "status": "LoadInProgress", + "last_error": None, + } + else: + self.query_objects[object_uri]["status"] = "LoadInProgress" + + self.EXECUTOR.submit( + self._load_object, + object_uri, + object_url, + object_version, + is_update, + object_type, + ) + + return LoadInProgress( + object_uri, object_url, object_version, is_update, object_type + ) + except Exception as e: + logger.exception(e) + logger.error( + f"Unable to load QueryObject: path={object_url}, " f"error={str(e)}" + ) + + self.query_objects[object_uri] = { + "version": object_version, + "type": object_type, + "endpoint_obj": None, + "status": "LoadFailed", + "last_error": str(e), + } + + return LoadFailed(object_uri, object_version, str(e)) + + def delete_objects(self, object_uris): + """Delete one or more objects from the query_objects map""" + if isinstance(object_uris, list): + deleted = [] + for uri in object_uris: + deleted.extend(self.delete_objects(uri).uris) + return ObjectsDeleted(deleted) + elif isinstance(object_uris, str): + deleted_obj = self.query_objects.pop(object_uris, None) + if deleted_obj: + return ObjectsDeleted([object_uris]) + else: + logger.warning( + f"Received message to delete query object " + f"that doesn't exist: " + f"object_uris={object_uris}" + ) + return ObjectsDeleted([]) + else: + logger.error( + f"Unexpected input to delete objects: input={object_uris}, " + f'info="Input should be list or str. ' + f'Type: {type(object_uris)}"' + ) + return ObjectsDeleted([]) + + def flush_objects(self): + """Flush objects from the query_objects map""" + logger.debug("Flushing query objects") + n = len(self.query_objects) + self.query_objects.clear() + return ObjectsFlushed(n, 0) + + def count_objects(self): + """Count the number of Loaded QueryObjects stored in memory""" + count = 0 + for uri, po in self.query_objects.items(): + if po["endpoint_obj"] is not None: + count += 1 + return ObjectCount(count) + + def list_objects(self): + """List the objects as (URI, version) pairs""" + + objects = {} + for (uri, obj_info) in self.query_objects.items(): + objects[uri] = { + "version": obj_info["version"], + "type": obj_info["type"], + "status": obj_info["status"], + "reason": obj_info["last_error"], + } + + return ObjectList(objects) + + def query(self, object_uri, params, uid): + """Execute a QueryObject query""" + logger.debug(f"Querying Python service {object_uri}...") + try: + if not isinstance(params, dict) and not isinstance(params, list): + return QueryFailed( + uri=object_uri, + error=( + "Query parameter needs to be a dictionary or a list" + f". Given value is of type {type(params)}" + ), + ) + + obj_info = self.query_objects.get(object_uri) + logger.debug(f"Found object {obj_info}") + if obj_info: + pred_obj = obj_info["endpoint_obj"] + version = obj_info["version"] + + if not pred_obj: + return QueryFailed( + uri=object_uri, + error=( + "There is no query object associated to the " + f"endpoint: {object_uri}" + ), + ) + + logger.debug(f"Querying endpoint with params ({params})...") + if isinstance(params, dict): + result = pred_obj.query(**params) + else: + result = pred_obj.query(*params) + + return QuerySuccessful(object_uri, version, result) + else: + return UnknownURI(object_uri) + except Exception as e: + logger.exception(e) + err_msg = format_exception(e, "/query") + logger.error(err_msg) + return QueryFailed(uri=object_uri, error=err_msg) diff --git a/tabpy/tabpy_server/state.ini.template b/tabpy/tabpy_server/state.ini.template new file mode 100644 index 00000000..8999537f --- /dev/null +++ b/tabpy/tabpy_server/state.ini.template @@ -0,0 +1,15 @@ +[Service Info] +Name = TabPy Server +Description = +Creation Time = 0 +Access-Control-Allow-Origin = +Access-Control-Allow-Headers = +Access-Control-Allow-Methods = + +[Query Objects Service Versions] + +[Query Objects Docstrings] + +[Meta] +Revision Number = 1 + diff --git a/tabpy/tabpy_server/static/index.html b/tabpy/tabpy_server/static/index.html new file mode 100644 index 00000000..52784f13 --- /dev/null +++ b/tabpy/tabpy_server/static/index.html @@ -0,0 +1,72 @@ + + + + + + + + +Tableau Python Server + + + + + + + +

TabPy Server Info:

+

+ +

Deployed Models:

+

+ +

Useful links:

+ + + + diff --git a/tabpy/tabpy_server/static/tableau.png b/tabpy/tabpy_server/static/tableau.png new file mode 100644 index 0000000000000000000000000000000000000000..72d41cd49d18ebd425595d65fb7d6b3bf4352526 GIT binary patch literal 33575 zcmeFaby$?$_BYP+2pFJ>-{1BA>0H-v@44@_>$BHdd+oK>tpSSi68G;B+(SY_x-TUus*Hqm3m+(F z-a!TaYN^qj1DCsYl3I>PNN8Ay?;l8U2>=B*1w>U7swpSSYiw)HXlP<<1ZH%#wgae< zkobjM?F@}A!BA2ouo=WgfNZ?@&XN5gQ13`uGUsIj=Zh{WY_KT0%gQ)CNk1%5~!sB*$YI6q?&Syq$0KsU{Vf7 zP6lIU7IsoD9!3@pPA(oUdQw*4Hxn~A6DuBZ8!s~}>CX?D;60$k?_gret1K%1 zvpe9I0GT-yYRAjOtM#j!o$PE#LUXX%E|yxFgUu|Kn-0PY#hmdH1dyj zM8S^64iGyi#MXus(XOG9trJv$j11Ay-=ClJvbOuXBOAw`*a0Xqxf3^*0sOn}1W>N+_+B!KHgTZ+2wZ9S znf|@fKUe&p?SVxNp#O(ZUb1e``R_)&yeecB27K z1{PLC11uc8%$zsdxIy{vZ2;n9VhA<-KWgPB?TvO!jCoCM9jpzZf)Hy%Gcc2#jTt}F zKX2Zk{F|wGMQp8X9e`nh1=;wS{?qLnwXPYBSIWi_YG`8&mJ$^NZZJY1CcHfCYyfhs zJPfAB#@r0-tj4SihRncEFb|tCH>W9&DVT-(8bmp{o7MkE)1tP93X(AGqk!vgBbn)Z5RKcng7w1B*YP5Pq!Ov3o!4sDpLX5|LfgblauKlzU}wfzFCn@~@Efk1X#BGL8?IkM zbd%sWTsP79W%)N;zl7)}!Ed;3qVdb}Z@7L5(M^KiaNR`Xm*wAZ{Su;^1i#_BiN-I> zzv22NL^lb3!*vslUzUHv^-G9u68whiCK|si|Ay%aQzaZ zn*_h%x{1aw%fI3JB}6v~e#3PWjbE03!}UvuZW8>4>n0k%EdQ@?-TUXOe_$KneLolA z#l9Y6{RhBHfuzQg%5q3Z?vzMKfB7OIom>IeRU{-QW+bFF5E2saTO=er+X#aeaU>*Z zFDcO%s;=W3^_GgN$4SSQzzb+jxtV!-`$iVDi+Mwn&b0n=p4~Y-I_AuWs`o7!-t6;p z*l)Y05GJc8r9-AXy6+16_G*xkjnP8W#8;OdZM-+7lWaDTK)xIKoQoD!R@Er(h!KyY zRrd&N+MP4wwrgN{Xkmn%ffKe3S9p2f`Lj{}&{fZsi_UXVG{g&yz=ss~8|ifcC?Z~v zM0~E_kwg?=e<1z>pNHr_u1LtYeioX3UY`T={PFn*@n4{yH~wx9>3*(6>CHH)~7RI<6f$YXZk21KgRtR=r0MQ zk?;bIz51_F+fx!K<%o;_&w&FY_+Lth$^0D9|6T&7PaKy>0Hl+ncTnH@fr`GM z=3NykO!C|kD)fpuAQ?eTy67x_X?Ep#0qytki&7F;Z?{e)}G$CsCvCR$Cvr zYq(5T8v?EWM7-__Zo92lxW(Onveu0eQg7k4{9a(DHWYh$=dz)s;$k$ea-#eyYd;z> z?wWPU5jwsQXhY5XXXC;xUWbinJ&BO|J#P>xTa3_E#%|%2^Y^_`HnYO%Z~lcLf{vFR z!D7pWK@0>9yqDG-$ zd_M7$eui5e?PQXj7fqQKu<!Q(Cq!BP2PBCE9V=IBMh>#%_#UAPgfqG1pSSCxbZ8`4rY%TC zGD5oz9_zevynD?}0BE?Adn+E#ELax~s8IyGX13Y77kW4*P&d^!1R|1kw5YNq-K63f zU;mU~?f{15GfAa43tjeJehgj5Df zeI`O&7W-6Te|TUs4tn}jF_tY-Xe5L@v9QDFh3)saocGQKEwY#V@!+N}V;hw7I{K^) zEhe9<#%}*m9l$tVPY`?cFoY)AcxYl9*FE#KsTnX9Y(G;X!v`2&b!AFnV}85SOqDrK zsEf;{AZxM%l~tWH%-&JpG1o}MY(6iqiyL-i-8CCuuVIi|e{JIty_!mzjT`IA(ox2N zwVf$BzI34Ki%mh=yh9c5O%Jv>vU2+?On7n0OLQE1Sh&t=#$=^)WXdBS}>{9jRb~ z^b8H3JFep=UlP}IT8>AdwIwn(X%Mp>%FToRi z_y&Z8w1mS1h?HC3i-d0!`X!iDqeG#Ho=nnk#GJgJ&> z>dn4m*})Z$Rso1WrNC*Dy@LBOLp3}~c_BYMcuU8|tLJMRUQOoSqpiAkL)WQ62at^T|AdFaZ(K*Vl-&FN$Z?+ld1rXVAs!@VOvFLIs9Sw$W! z&@y)tY&$eGUx$j+6>vw~P%n)Ue)qM!9i~idiAvo!#eocUoCaTn(7{;nEUBLH(~rOJ zDy|aBdlO%tJ~Z$)^S#GpWIrMJ7X(W>k;#4tNAV{&FFG9_XNgwMs6!>oErjoV|JH;Dqs?6oFe>X^B5IJ*?6$7WK#LsjHGuehduRS5V_Dj5vmk`UVJUaSXoEeJ zS7SIGlp>#9=_piMXu2chOsRPlrIs-+#}VJ1u^Xks7P*J}*AH)%STGAuVa znIz98@R)sID%t3n(TwM@LAU<9Z#}>k!A_JHobQ}brITM9CN}S@Vsm#KAgBUC#y^(!BeqIDFei2z6?%rXiOVVcvMfUtM~|1v z9)_-PF*fx0b%Y_}e9%lmq&k87W5T5)1fuL4tfuV`iT20A{SW~#;1Wa7lQoC7j? z8U4B;lj+m^Lyn!^gT%*Dvy_CputZlSn_60JwZ5?1?SO8$W~h&Yfh}w`(Ts}W?r`)| z6{N_z@~G{zqK}HJX&gAUgTl5M6~^D!R@k`3LrL^YUPraSPnAlOz&OTPhzrW@Rsc+y(}o#;sc8b*BjR?ASHi2*8IrAI zbjG6`K7X!=KEv<$Q(Oz#;_d@I`PSk;F_`f2BB_#Pa_hvqd4BAOO@Og`UHb78u{Yj9 z1@^ALe+FspWT}vxR~FVcPMtdCk6i#?FEf{mj*q+loubw+mTB9oQc~ zwpYZ?n%V3^q(9$`DiDmvltM_@Z(iqmSC2_PtvFSc0JxCwnq_`}f`IBF`V+cq#q!6} zd(e5WRxF!#f_PdBguOs9N*s&ufF1ZbL*ffG32gM&*doSf9O&uOW%i_!IY&{{xQsj4 z20NqQ+@9c~ca=jJApF}H00zZUj-A6;xGTQPw0NgA;*E)wChXq_OUwyVsx{Tcs_ylE z@WU?nVL$wU2T%?-fBqQs4~+hSU>n*`I|QiR#ne>gbT-i_=rVhnT56y_-{a%v)mM$C zf#-F(SY8z{F>{HR5c|&&&Ij0UuG`{<=#re(hWG6@o~^Yu&luG1PYK^?ox}=?Il>x4 zjf@Yntc=`s;#zE2dYYCKr#Y&e6Q0|iH+`Y%CU zN=;urq$KcjEyh`V30uw_avOr~+_||@{4bn|c#foJx4aN1rFE|}$VfQjoz^GVLB5+_ zW2mL^=%!qc4O}9h_K2XC`ad9kfS9O4)XM?9AaJz0GDlae^*W3L|_)@FIPa^`!kk-%kI; zlt=eI)U{hNasV+WIDpT$dWa7$mW9*cFp=1?LyD`$k05Tb6Mk1yiWB$_w*~1R$mE=` zzC-DTSTQDG)|m|WJ{$W4joXEK`+&YAI5aL2Tox)!bbSEBOgZk_nFADb`mN*#43-xr z&n=*sD&_@QvRc^98yyKLJ!y0a*C=rN{e&h@To-YjFpbxgO9|^5(loMV=S00Ih({G|E_K1=+*1;!(7JKYviN zK0m{lP^`w{ur zNJ&S3FMGWAHlWWu0sZV?x8mS1^DZ3wnPhm+nZFU8SY_kY)ulFOQF7{b*J6}}+tw|j ztQfWJlTqFJtO4E4Z5D&%@1{=?LT2_ZABMRf&jgfrUwsEo3nd14^#f0sV)VZu-(IDl zvWMw;4-D7Yqp1gWmgS;}1wVgPGQ7#_`E9fvELuj|rabc{K>kq9;G!W*6HtA@u1$}5 zJ?w*Ks@6r|f-aUARAZmu6nrgr<@*CNn$gsq`H}hfyi@7*0o|Ju1*k{+h)CBa;*Kv< z!+JCQvA3GuPI+Py&!bf|80^9qaeQT|%K-a*&1Iy6nySR{sbJ}X@bLKfXNESxc)a7o z1~zh3Ia5|4RTZ+g=o~aZ-1b7VmW(v25|Y3&)leacaP$BZ$^uXPj~o2RBXp>R)-I@ zGkF)c)HZo+^bH|9jz&_h16xAa2zk2JGseVN_;pdK2BrBD07s0skm#I`zQIdqzS}L; z*4u)Mlp4*Q^QyiWjd6~Wu;p$yTIX>6RGh`+krZH`;SD61hMx?IGb5XsurQ2CQt<=; zHQ|i?cNv{twzr4Upiw!!pwy<3$K{hR!=j4r6*A%Z1dp}l*K(l_1HjTtAbPy0p-FIF zXM#qv%uQ8IHUg)Q92a9ftL1yAnG3gwb&$PgQVAh}Lg zm$ffO3NQkn{(ubNI}Eq_jVS`X7=dQzk#bhj##xh?Tw%TM(@ z#=@%&63$CnmX|4YB%>iGE=w8x)Ka~fk`NSf2d^meRFQ5HF0HpJm_dHeD^je;AP$|H4@X63?g+RR zYXzwh7RR+1weo-4_(Gk$w;uRX^5PX;Nhf1bm_XK69wa2*^p7V27FDUC#ovPTvCWe$ z0Vp8k{0Ys@$U}3^JPhKsUlyo{%;d2yl*pgL;0$@9AL{NfP|BD2VL;IteL?(gmu}W_ zbX23CEt;kDDO#n3-W_49wWXg05V{9rm{KELb25yNTeXxfqe#z6sZpT)b)nYfI&Sewe>6%*@>< zwfk=4yMtB+y{&F{{m5l@-5s+C7L98;pnc5p^=vE({KwXYJoPJ3TGv$zOX1KaQi{pYV5z zV2dJ+5}#6mrph%ijhaZ=<&1n@pl+?d%#Z9?P^F=30gEfcm$U=1v&3uJb3k!EZML zvBvZnY^2J-vrt)#TGpfZvK*!<45+^Pt#P3&3)T)7FU^lK87V$rl*w=z4=pD^sH%@d8T>!meNUsudpl1y+2v`Y`D8ORM>HV%EHTvm3IaZs;$o{~|CXdo`A83i${bMV%Rt0=%l_Rkn$mK9f37kn-^uo8GrCPqH_@Q`}jh(k%r65xY5qGXwrN|4vQeVEc*HMIzHEcZgdR6)f*$3h`gt9s-Mn;Y5c8|k&d1S~+6 zS#RPBtvc!j`7GsNyD6wKW53YWTd-8MV2+Zj#RH78iK=yS8HsH-0VZBymB$_Tn%-Js z1_7q~J(os_*R{ue?lb!CwHM!m#F;N23GPms+iUlHXMfBfzy5ftXd!ZUjxt#mJEIRc zS4~k%rv(@Z1mj#7$g&3Zo0oCtFSnu|U2w_tyW3!2mAF`mh#&hV`Q%X_Mh46U*({n7 zl@^8iJmxvvkbP1)eDprbaxbh&RSq~?IPCF9#V;q7Vxns?r{bDL;*mN&o)VFF^p2qS z&Y4SBev|K9AD*xm-DmilSp${q04(q_f!@N`0_@yz+AZ&L04z+XL@hqfLM7>&w0g@FZIiDNIUU3&3z#2Ij*MFl7VaL{MKI)*K@;I((~erq-JwAY z(wAy2pr#lR9EmfsCybE7sOaL8#Z&dV+TR*v4aVCEBxw)Z0Q=D}Ly4)R&YVjEB3F2g`O= z$Z1)&-$^<@Jiw09D8K$%<=GTl<=UuS={0OBzBs86LD?w2g<>1UsZOwGK~2^3l?>k$ zV{xH+^}eOoV&=iEB0K+&d@KyH9tg^&BS%HdC&) zqtzW2k@|i?knNA6+*OnW^92&~1I_R2Zq;AIG2 zkZaQmKq2B-c{R@MH77QDMUAYSZ5TcIe{d7hNENai#6snPkuaA-zl{Qxdt; z3ON19I6dmDxAKn(Baok)DB}0Zl%G- zmu?q@4v*B!{3Dj-B+CrM8clY$yuMu-G_T^{rrZi^#u5`n$ zN0?}f^XB$7^mk>CO6y`mb=>biwfI&N}t*C zS&zFH0q$BK6yI3?6|sAAGHKrW6<9jbfZ4}BACt6-i(ycGc-X@_Y_+;18Z(`?l;7h^ zfUZv)+&<$CXiajas8`$p41;s8zs%6e$K0<#0z<5b>OZ^R5f7T+QSdxtV=QIQlTRBN z#%gzH>%XH198`+RoX=(@XGh&Z^$d+4PvHGrr+2hYUB%Nsj4AZzR#WvJ?c zau6lJWN|Jd@!eD#GL8emrnGwcQcAnUQmDu;>8CiHd*cgzx%=7V#9BA5`dOunYF~~s{JS);DVh{TL zRg`6JILarY09kVuk1rr+i2(nng?>SWmal{{qxg?>KkVnWg9^yfXsK+DIIto^gC`zn zlj85IFh@=GWgmq_h)jv8P6!1ux_i5|d_?Tx#wVz`STZ#Fc3;t&zKuQ!v+L6KESMT= zMLP;3uw~E%su_2?tG|-iZ|Y~uZ*1S`bB>Fy0xX& zHHBuh6B=m=685l2F(eKVf_c{bW_UrbRpP-^pND=A+Hlx-oVJ~E9mrT@r;N>h)a^;> zQ>)fbr|yHGt^`mQUedV1#wG@zffBFqRI(}U)z6+tot%$hq%zRR~Ki1Qi7Xfi;+DX-}V3nMm@p|U*~IO z$bpt>D)@vbA&YF@13RzdZMRy#5dMs$k?QH!=b``VhQG1$S?0Ed&@b=rrtA?5O-78_@+Ej z*a2~|$fA@lQfV5l2pn^*+p8>BD=|6@R`NyVar1r3s1g%oWtlyzIMXp&?wM~Zd>`T2 z+|7pdP49NN&rAmb(tj4Qy{~~g$f;2`JE5Vaxt_)48ECb^q|x9k1cY@p7ChqVTu68b zjGMR*H(c$}$?zSYi9yo|9pf|H4LqKQP7fR@$ zt(F&9rvrHRWFJ<1!1Uv=NKf$#vZTu5uWqk$=F4YQlW1dfWGVwFyWcd4C*b^P3?{Gq19BPwpQsh#?2dVCU;ZnI|KgeiSYEFA>>rmumMZb!FAEkjJN5rtnp2WFRETpY~^{rGuqI38so(iF45hXx(p(HPH9U%(xIeYWn=5*#Ans91Dqgjl4eg9`_P1}dkz@IrFLDqR#H+DU zJvt%CB|75@CY5Ek7e{!)r?9w>Lmvj6P~qPWP3lso67Kxmh%2Se*(LMlAaohogL~Mm z6k)oM*I&sjiv>o>(^JYjyf(h5S+&fq)e4kK|E3}3XiU^7{RVGCwJ0OwPZ(y}+*ZEz zVgiUt5S~G|D;%J0n_}JaWIeeN)R_{<9Z;XdC__w|)h%PEC=H@ZUA9!aN% zIUR?6$#U)0?^^6HU&m+>8rss3LHO_<290LmmRM%0Zq{XD>C%_Hf52DP?_a-c@Mdvc z9dNb{2Uxw;tpmZ`1*}#4ZnepO%{5o+>>C)v;1%L&e{r=e6Mtsxouox@E*TC3+z3lK zpys&RHs?aaz5q&x{AQ-bs7z?w==&@o`x}Qk>`~h_dk!gaaL_y!hN|@ z7XAc$*c5jv48i@ZR3nWm3ty5w^8zAu%RcRe)bJFiW_Jq_sRKx@A$(}^)k*X*9i)x9 z0*R$-<6}$PT3pd68-#$Z^3xS&JiK?o0*-K@5R+cTcS`#ndmw;pJpD;3gqbN?SvND0 ztmW&2uO_;DC_S&1TIATh0@pO}dT2dVp5 zXxyK|{l+6kUoojMtR;5P&2}ER0!Wm7`K0uzp5;KCu7)*830YCGy z{RL4}b^xc*pG~gp^o@MFzyK2#64aKDbaButuV!S&TN?ViwW>{{wk!iZiv~MaC78(y zXTR?rrdq{)aoBh0<11aO+U@sM`A!_~De`l48y3tKBh2T@e2O`9F+5@)so=hjA*D9%uCkcl^_hd+90JVc)BE` z)66^gJ{b&?iJOsnA1lfYJ7I+)aUp7=ZCsM7idaXfq`z94YUDN$+^i&h7YEXP1(MZa z4v8lpC?!hYe3l`KKWc@paUIqv@?Uh%>V+yM`C{c8x}{Mxqjf(7Y#BJkI;DP*Kj3Fx z618-#7y_9TPK|L!KSTZz00=4F_bt#stMjHk*vf*9ucEbbQ7H(za52Ai$hv~=xx7@N zaq-QH*go=!Isd%#+RRj%xvUS~u2N%`3m;r2PqV-9#f$XtXJpvob7Z09hO+Q{*W%{c zImwW49}sfo4*QzvRQiWU2ZimO*1?+1m1k3RX2waU#@pE$j9YqZe@c1i5^LP|3t@=c zdbjmQlj{MB)ZkVH0Z}0KpJ}MZr-K`-?!jUh2^P{(B`;DE$G47L-r4mG-6=95m!}V( zIN@Ei(a8v>4l_YZBzn9|5(~;M6{g_amZa)=K1YF{Cs1WGGFg7N7PL zHpvc8T3UE>f*4}T4{=}|{c-BF88GX7oT_D7z4W@^Jo)B~fbZNxjr9&aZci{d)tE~* z2%7A&^(uj&6e2*-l{sHt(mRq~FGNd-T*t2(j<>;5&v^<2Mc+x|m7XXk-)VYl8i*|u zYLR(fQ(v|(c}xzh`^(Fy%XH|PjBMI-WmFU_HtM%Uul|(EfpVHzlOr1|Qw8U-4rfT% z6*nw7SbRw<3$=n^m=b=SXb0^Tt)L)cm>B!d)CVcH-6sYE#_9FiaopzdWU|wa>xf}2 zwJ5%L8!2DrhBLca`yi%4`~}d;lV|uB$*&`w$Shs*FYDrX@H)^RLpDP`S1q%4AogYG zkn*_ihj_y9{m5LB&wo7^V;+>uuhSd>j}58z=3gc`m9oH6hL25(PYJ9Ay6?$|bQJ_o zr^vub5{w`jmpz}~07`$WXJRw4aA9N<;00?&rflni(UYcP3A~_7mG&l4E&4}?gRyVr z|KN$(eLY+G1_b>76D^0{`qU||#ZiNjexA4Pq6>P)sdk1VeQ09dB=xQKLxiSfi7f>B znP&O@LHutJ%d%_Ko(?78e5qM$Y&y2*&4X-wbg%Vg$UrCHHg@87Qhryn-yT5O9H1`E zrS2rcY$THKt?vDUHwk5EJ#cWx!=4?)NzHyH1LcsRo@czV^eAp5GrGRY*=%ASHz>R8 zh1;xdxta5nr8?jTjIz`W4hcBQHpZDC2+x4KBy=3yK1YR7L0Q z14d)Fz9Wyi-P+M_%H<5RE<&+v+C94C~dujTRf=c`Ih(x2#cWz{I3-ti^ z8GEIK1yB0$NJt@0874Z%VqY$eY#}@mNFs}9Q%UVD0*-IMX|EP`-LNk zP0r3_vjv;=RtQu0m@Q=O6Ktb!2Z%@EI7 zjux;m?G2o31@sG1{qiBOe@aL#`T=?2+oY5d%+~|&c#kC|L-R0Mx=B>PD`}tbPOCy) zibHBbLl`7pqZALw$A&{cFWD^0$xy{~*q&yjL)!!JMhiCfuapImn%;WBia+&#)4(-L zz}I?S9P?RS4S)XCeY}EDpW25oOhR5DP@0?YS$FWgya$C?@aCXD5W>N1A=M6B9$z1( zS({(LHi>7Ce&#-;i;^4tGFdi&_D$&L* zH{jRO$(0l|v~GRqQ)feK{mNgWK+xCmF>JTFijX_;k))*AsuE&JBvpixfWve@VeUxb z0MtdPMysphC>$`a_A9f;_H7I8-({=`t7b`2eoOySwl5ZOp-j)05A158Svdm~J(7wJ zmuaX)#ciF`@}ti?;ehPpv)UPF-UG&U`>KQRg25n;gm0;7e!q z$Dn1iu>|w`bZDOPWu@+--3boM%r*pN_=Tm1-lN+;4vEf6CDhVT$HJCsjWv? zW58Benhz-Oiv%(Sq?A+dQn;@a`0~mhik7X7yWYZQ+^jUW#oXFRStB ztLm9JuYq4J5TVtH8+1VgM}U+9@yv&oHGa)#4yS+e*@_fP8-(44wR^2_jKKPp)rxv* z>Q-`=SV{vr3bNIUJC=3TA+k$>=a-}3ScwQ5GMCOQ2dDr?mnSOVtgd%CD7|{n^oBvr zBP4#PKr(KOe65G6YS2fh(R2P|WLVHXxpAqsiTCaYIbVw`Ho5z#e17tQovLeo8?!!~ z+vdS)HQ^EWGJ@}jTAO~aH5;nWOyMX}rW&VyQT8ETAb|qIuFJ@w85|t}mYFEqZX0Rc zrdLg~PvyK1Z|@>|3D!b@T;vDj5qUHr?7UrZd){EYEQHHH7%!vGb@vhCFw!^2etI<8 zCA;SbC{|N2R}KRMPQrpBUe36%@R4m6{o@Bfo8J?^MjLHgumdSmOYm+8mVAI3g$zN_K`V7G^E0cX#mq zRn3U+(r~qJa51tNui%gxJ%>Wr0@fxk#a(#nu;#`vM!lG^_fmo=*IlULa^B#wV{@C8 z!5cErK31WaBzu@7&jy%3b;`a^M<~Q=_-XnoSp)H`@D9XX6o9;wJt&F@-?|x)b~F~Q zlmw*3=m(0%mdg>&QDl`o#pcC}OR8$nAJ$F#l2jocHKmcTcait%Ud5A7x_CgiZ@xA} zuaFv|WL%;e6qDH>e4ie+D?)W`#w%9w?tbl2)*okozi?vV0_F*jtCf4Wxn{)q0@}ad z$#q|qi-^mBy9~Y4&sz5R_Ey26CSEe`Wup>4gp9ctoiD8JIEIoLj4f?iTSP%a?x124COu{nHRzEx3Bua| z*y<6}C{5@25qXae#Qc8V0Mmu#^SLPRw+RmJ7DU?88obZl&6cv<~$rOfG;^miZS17Rwj zWnJVnpB3$rM@_xX4$1=!3sJYK-wJl!J=(d~$#)6lTbz6>9_T#n(UnFd0-WAvdUo5N z@fi?lDz}PnG@H84cL0ww?P!5hBrzN4`d>>>E7i`L4-~sL39f4o2wKBlh@zEr@jalU z5v-xNG*Nn{ou<6_R_Ii<6+)8LdR{>xn%h_(_w*zSM|jjsQqJ&^3h&I*L8>n~ms$h+3AdeBR9g0EEn(VK(@kG zRVkkb<4%!>8VC@lns&c#*49}s*-+vO$<_R_sgi7<&tkXiPjYz=w)=fRE9oj$?pDGA z>6Fme;LB)xBdr6cr{?|8sF`v?V(u`-T$0-9y96-A!INoDGdB0>5WHUrHAwaAEPRiml8 z=K)_@o;*Y!mIPUbRwNcangY|vsq=i=crT{K31o_7gVxV*+5~xyBg6e1Q~JczE&Jdrqacc-@&8NWI|h_nFFI^d2M4dsN=~d@M1P-eOZp?5onPva&?ysO1Qkg*ggK z6MvH9O^9j77iiykO3?mLB_d79_abwoH-)C+zQEl@(?uFz# zbhm&zLeb>qY9bRJWIR|hfwe(!v!x&K=}2k;$HFp1T-q^(VQqi{U$pGoG109~*FK;T zp9D~rYZE1xL|BjI*BLpgXfZj9VR^lg2XYG>SmRDgk8IS$*j|9hgeJ!G1(_|hM|%<3 z54>r@5m{<7;UVU0Mw+z2QEGp6R<${-W|Fcw|O4lMmt?XCbFj<}S$FJ#4HtJ~a0%r^f!r}XfL+TSm zTJKxFTKIjrx|Mr0{U33~^Z3Fm@O7q~uoHyj*8|4wU2+V))DN^jCCN~yN!q(nL`zbQ zn_m@|UU?M|aZzrHoFtOf1{O$R5=o@v>*_1!)D|eUB*@DJ2D4f8Wo+13;wGlh?}vOg zYb2OoZob%|!9u5$V_eMOy3MUbOh(oD`q7Z0w}w-8hHB>OFcwi$|GirEZ&V5&mnh@o zXt_pA+aC78VENWKLh`pFZ+8HGzlv|seCZI9NGt5e?Cl|1x|vVzE0GEGjeJSKLMPL< zdp|$O3S==wd8AdiA<2X~OgrvX*z})0iF_y#)m3~S!#^QVUD`e`lP;?MfdepiNE0{9 z&Xf3s#1}6q1P%q_fsCKeHSSN+q+m}CMuE&Cxm;hI$hzc*2_Os!hD&6#LBjh~<21~! zcS|J7-XlUcK>pm5ZxnbfXq+4~8rF0)7+3?t8b!5fgy~J*83j@A92*8Dn zn3UIC{Oc>L>Ao7^Orv}$K2DI9{kkl|f?Aw_$h_z7qI+NhVPF|=+FDwaSQ%hhv*T@Z z=v_H89{L#Qd;66aI*TheiCPYlX}N@o9e5Se@rtmsjp=<4tp}QmEbR@Msqw4D{FPUGW`NRxEV612UTE zQ>=$gRU-vOC2xWkK&35`Am$*kO)PCBivi5RDc7dlE%&D|%&AWmc9xbrg%6BME~!s5 zmm)h8C7}d;Zz9?GN+bKTyrZ}nY8GJj9^8utoEV;%NrZg6+KA}vz}d?8(O8gfNShOa>JnzHk8{{vd!h;eDILFH;rGFEdkQ4>1VvkNR7Y7Y-Q6- zt`WxQseG6n6lF|MQo_inw-|&;1Me|K`$fby@@Q6-m)Cu8;dWvq`CvKkf;(J!v{a{F z;X_4t4)$${(vzU%5U2SP5{cGW-gmd0Ss$zV8!{i_4Y7bDyS=@=CWKqVOrrIK$3w`o z14SeSNIT*I?8A7#p2%?fQw9SHSu(Bi9?aA_@vzh;w6tKi%HxYy#qD6eEU0bBU|NP$ zk4&Ci_V}okFIVx9yXaKPo4nqa4lkvBv=@ZAc|pa85JjHoz?N}mfO7YOngAjtWbX2hpSRW2N6rqq9xeF#k6p02=_|i z2+fLH+hjH!-tn#?r?zxYmF_4MZo?S{ioq+J9fke+C3v`H9grPY1-SWuoY`pK zI*=KISG2s6w5YK4v2N7)A$VNvABf+&{ZlSrI(~@BqK{aU8-&_Tkvm3Xu_> z1!N;F@`ya$^b+;%F%C7exsy+CVMRMRu0HKxMNPO30geskC-Hk8 zaXA^#d@u`Kb(vX=--!%O;U5ANuN$7nHEG z=Oey|;2qBQQHjMFAg?dWt@uncmC)D)_vnGX*13PfEFQmHVkY9Hg%1~cOV)C2!_^l8 z&lFU$R=*X$+?Y<;nY(-{>JPjZZD)S)?y{+~UE<7-&Eg1JuMTi#wL9=C>iCRx0Wg+= zv4O*J|9XS^hxq7Ox+JxwJ2{hf@62GBIKaMHyJJ3c5(EUomJmD82f*ue{hpiN;Ajrqx$;u1ybE-}82cOdALR&Sr=0cb0Ggxx{Bc?w~57K$B@_18MJ}wY~BTYN&TwzMb57Q;-|ByMK^<8gN@iocEAQwDZ-TQh>fJ zv27-Wqh*Xv&4XqNE0F7WM zI|7LU*0ut&6F^ZA1;k1lDt#4r5F!vGu|U}Zlue>#aiLmd)3huFAz^uhB>@^9mVHU* z-ca8^eRIw|nK?Ie*ZKYC-kI;`cPGL5G5vX)?(xM!bWI{A?5F#IY^zv} zAL7(ha!!Y&aMvCDlf@yZUbU( z_el0RC86ap-N6(>DBTF0^(KOX448fm zlrx?xHqx!}_0wCySVSrA+GTBO^b5lrtMsnUkxv%%m-=8C&!VlV#P3VmD%ocjHe|o^ ztRzNZ1K6vt=hO?MQy+ca`e1bq>qE0+Zic5)%z!R(Mg1j*olO0s8bNjlPY-Gmo~|$R4|acL6QgsVnOOS<+^zKiL*B? z$&sng?=0d)5MKs$^Pho5Q=K51rd;4T$dQ|~aY#of zT!X*VtYhsMz8Ut5Y4GAFbv3Ln8SI0YWj8R0OA6-b#J49pf(smWDm6rusW!5;+%wM= z%$(WCoLO9vr&o;S%(875Rr8Rluy=s_XZ-23^M@NNw668|25hC*IzljW`jyEB7x-=W z`}6)hAj*RleLKrsef^fo_+xet6<>l6thY11SO}WgZB^kNa-rc3bsj9Equl3ES6FJHjJWj<^Iy6safm-+ zn6*jI?{=9G*8FD}KH<>|`EgS`u(wOvlrqiU}hK{a2D50}ylU#MKWt)m}p zZj2dbr>d`j54wY-j;NoAU2<+DTsDhSTEWy?b1=F2ea#26gOWPeTh3(#>Mq+gHMIO< z5lRh?qWtF@pT6)n_|pjMu)4&#zK}xl-bNH%L8BpaSoQ^PM-P7Z>v736$0b_M>A6?A zJLE0Jc(N0QP$}+pDRqZo{&z4Ts4t}l;SEVO{BRa}_=k#c;T`h`O7JA1G?0Omc98zP zoiTJn$CjqCz2T#bjsW-5^gYd$fqQp8*|D^vh_?%dw5ORyzHJ9gA94iT0d7;(5(`bz zMzA_hfML9(E_pz!f&{Qnqc6av?Gk{ldY{L2k*B{SG}&e%WE0KvUjG<4s(U>6rzdk0 zlMbspl;o$x)-rnrzt5m|Z0B_YUf!2O_$B3PARcgrb6I^I-Dz%A(b^+}6DW@UA3d=j z^9uMQ203aE*TNq?R47Yx+xO<^BBn8AGhJdHK@Dcd0Ho=^RC|rb>i*zfK}1Z)#$GfA z2RY-*gfb$QwNFW?<<(mh6e7QZD)$8wfRR}ey$6b}F)627&UX81&6olqyHOgO3Cqh2 zEEqhdIF1ek&ed;+hX(8JRvFXOrp+IGi$EjuE*;l;aHpyuLvOo~f(p>qM3Cw&ogoMn zBstl7z06&1F7PTP(Pku*a$g)4Uyj#R7}?6VcgF#(bePp8Oq$0Q;~pXze3&7}i;BI5o1kJcSSI94)^&-XW6BA9lg%F$YFeAc7ag5}1&hd@keU{=Vp6wSL`? zh2C*6&T_^%Ij$h9I=f9Y2izx;`Ud&$pn=Z)irSAq+iB6ii)+T()ZGSVf>w(bX=V87 zp3io+`Hb&+AmghVKIxFXLMU1n@~T%$(rnv32wffk3o_P44cH_cz)OSos3VqU=YLQ! zyHuC5ST!2iB3iv$n=JA=8!ZZKonc$@2}|vb45f~(9i&z!9*z953!KjUAOErvG!G_J z`qME4DxlzVPo8-W4ka-_)suERc!&)oHD=Xg{ZZGpTD{JA2h0G%FhUw;BD}?VmGAh@ zRF=M>#7Io~b7pIw67O#o+9m#A$o1t8fAVp1Y#ncyGG76x-U=c+f099N4pK(9G{(J4XsbYfU%p(m zmSglDyfsi`5{8Il!Mv_poNW%QO0zuyuLKT4`U}oMn)ZAnC`jAYUO)qACDez&Y414y eA&}DqdrI=Y%+0O2qfA*8ybBK(l5?$N5cMw;M9ZB3 literal 0 HcmV?d00001 diff --git a/tabpy/tabpy_tools/__init__.py b/tabpy/tabpy_tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tabpy/tabpy_tools/client.py b/tabpy/tabpy_tools/client.py new file mode 100644 index 00000000..f9e90023 --- /dev/null +++ b/tabpy/tabpy_tools/client.py @@ -0,0 +1,389 @@ +import copy +from re import compile +import time +import requests + +from .rest import RequestsNetworkWrapper, ServiceClient + +from .rest_client import RESTServiceClient, Endpoint + +from .custom_query_object import CustomQueryObject +import os +import logging + +logger = logging.getLogger(__name__) + +_name_checker = compile(r"^[\w -]+$") + + +def _check_endpoint_type(name): + if not isinstance(name, str): + raise TypeError("Endpoint name must be a string") + + if name == "": + raise ValueError("Endpoint name cannot be empty") + + +def _check_hostname(name): + _check_endpoint_type(name) + hostname_checker = compile(r"^^http(s)?://[\w.-]+(/)?(:\d+)?(/)?$") + + if not hostname_checker.match(name): + raise ValueError( + f"endpoint name {name} should be in http(s)://" + "[:] and hostname may consist only of: " + "a-z, A-Z, 0-9, underscore and hyphens." + ) + + +def _check_endpoint_name(name): + """Checks that the endpoint name is valid by comparing it with an RE and + checking that it is not reserved.""" + _check_endpoint_type(name) + + if not _name_checker.match(name): + raise ValueError( + f"endpoint name {name} can only contain: a-z, A-Z, 0-9," + " underscore, hyphens and spaces." + ) + + +class Client: + def __init__(self, endpoint, query_timeout=1000): + """ + Connects to a running server. + + The class constructor takes a server address which is then used to + connect for all subsequent member APIs. + + Parameters + ---------- + endpoint : str, optional + The server URL. + + query_timeout : float, optional + The timeout for query operations. + """ + _check_hostname(endpoint) + + self._endpoint = endpoint + + session = requests.session() + session.verify = False + requests.packages.urllib3.disable_warnings() + + # Setup the communications layer. + network_wrapper = RequestsNetworkWrapper(session) + service_client = ServiceClient(self._endpoint, network_wrapper) + + self._service = RESTServiceClient(service_client) + if not type(query_timeout) in (int, float) or query_timeout <= 0: + query_timeout = 0.0 + self._service.query_timeout = query_timeout + + def __repr__(self): + return ( + "<" + + self.__class__.__name__ + + " object at " + + hex(id(self)) + + " connected to " + + repr(self._endpoint) + + ">" + ) + + def get_status(self): + """ + Gets the status of the deployed endpoints. + + Returns + ------- + dict + Keys are endpoints and values are dicts describing the state of + the endpoint. + + Examples + -------- + .. sourcecode:: python + { + u'foo': { + u'status': u'LoadFailed', + u'last_error': u'error mesasge', + u'version': 1, + u'type': u'model', + }, + } + """ + return self._service.get_status() + + # + # Query + # + + @property + def query_timeout(self): + """The timeout for queries in milliseconds.""" + return self._service.query_timeout + + @query_timeout.setter + def query_timeout(self, value): + if type(value) in (int, float) and value > 0: + self._service.query_timeout = value + + def query(self, name, *args, **kwargs): + """Query an endpoint. + + Parameters + ---------- + name : str + The name of the endpoint. + + *args : list of anything + Ordered parameters to the endpoint. + + **kwargs : dict of anything + Named parameters to the endpoint. + + Returns + ------- + dict + Keys are: + model: the name of the endpoint + version: the version used. + response: the response to the query. + uuid : a unique id for the request. + """ + return self._service.query(name, *args, **kwargs) + + # + # Endpoints + # + + def get_endpoints(self, type=None): + """Returns all deployed endpoints. + + Examples + -------- + .. sourcecode:: python + {"clustering": + {"description": "", + "docstring": "-- no docstring found in query function --", + "creation_time": 1469511182, + "version": 1, + "dependencies": [], + "last_modified_time": 1469511182, + "type": "model", + "target": null}, + "add": { + "description": "", + "docstring": "-- no docstring found in query function --", + "creation_time": 1469505967, + "version": 1, + "dependencies": [], + "last_modified_time": 1469505967, + "type": "model", + "target": null} + } + """ + return self._service.get_endpoints(type) + + def _get_endpoint_upload_destination(self): + """Returns the endpoint upload destination.""" + return self._service.get_endpoint_upload_destination()["path"] + + def deploy(self, name, obj, description="", schema=None, override=False): + """Deploys a Python function as an endpoint in the server. + + Parameters + ---------- + name : str + A unique identifier for the endpoint. + + obj : function + Refers to a user-defined function with any signature. However both + input and output of the function need to be JSON serializable. + + description : str, optional + The description for the endpoint. This string will be returned by + the ``endpoints`` API. + + schema : dict, optional + The schema of the function, containing information about input and + output parameters, and respective examples. Providing a schema for + a deployed function lets other users of the service discover how to + use it. Refer to schema.generate_schema for more information on + how to generate the schema. + + override : bool + Whether to override (update) an existing endpoint. If False and + there is already an endpoint with that name, it will raise a + RuntimeError. If True and there is already an endpoint with that + name, it will deploy a new version on top of it. + + See Also + -------- + remove, get_endpoints + """ + endpoint = self.get_endpoints().get(name) + version = 1 + if endpoint: + if not override: + raise RuntimeError( + f"An endpoint with that name ({name}) already" + ' exists. Use "override = True" to force update ' + "an existing endpoint." + ) + + version = endpoint.version + 1 + + obj = self._gen_endpoint(name, obj, description, version, schema) + + self._upload_endpoint(obj) + + if version == 1: + self._service.add_endpoint(Endpoint(**obj)) + else: + self._service.set_endpoint(Endpoint(**obj)) + + self._wait_for_endpoint_deployment(obj["name"], obj["version"]) + + def remove(self, name): + '''Removes an endpoint dict. + + Parameters + ---------- + name : str + Endpoint name to remove''' + self._service.remove_endpoint(name) + + def _gen_endpoint(self, name, obj, description, version=1, schema=None): + """Generates an endpoint dict. + + Parameters + ---------- + name : str + Endpoint name to add or update + + obj : func + Object that backs the endpoint. See add() for a complete + description. + + description : str + Description of the endpoint + + version : int + The version. Defaults to 1. + + Returns + ------- + dict + Keys: + name : str + The name provided. + + version : int + The version provided. + + description : str + The provided description. + + type : str + The type of the endpoint. + + endpoint_obj : object + The wrapper around the obj provided that can be used to + generate the code and dependencies for the endpoint. + + Raises + ------ + TypeError + When obj is not one of the expected types. + """ + # check for invalid PO names + _check_endpoint_name(name) + + if description is None: + description = obj.__doc__.strip() or "" if isinstance(obj.__doc__, str) else "" + + endpoint_object = CustomQueryObject(query=obj, description=description,) + + return { + "name": name, + "version": version, + "description": description, + "type": "model", + "endpoint_obj": endpoint_object, + "dependencies": endpoint_object.get_dependencies(), + "methods": endpoint_object.get_methods(), + "required_files": [], + "required_packages": [], + "schema": copy.copy(schema), + } + + def _upload_endpoint(self, obj): + """Sends the endpoint across the wire.""" + endpoint_obj = obj["endpoint_obj"] + + dest_path = self._get_endpoint_upload_destination() + + # Upload the endpoint + obj["src_path"] = os.path.join( + dest_path, "endpoints", obj["name"], str(obj["version"]) + ) + + endpoint_obj.save(obj["src_path"]) + + def _wait_for_endpoint_deployment( + self, endpoint_name, version=1, interval=1.0, + ): + """ + Waits for the endpoint to be deployed by calling get_status() and + checking the versions deployed of the endpoint against the expected + version. If all the versions are equal to or greater than the version + expected, then it will return. Uses time.sleep(). + """ + logger.info( + f"Waiting for endpoint {endpoint_name} to deploy to " f"version {version}" + ) + start = time.time() + while True: + ep_status = self.get_status() + try: + ep = ep_status[endpoint_name] + except KeyError: + logger.info( + f"Endpoint {endpoint_name} doesn't " "exist in endpoints yet" + ) + else: + logger.info(f"ep={ep}") + + if ep["status"] == "LoadFailed": + raise RuntimeError(f'LoadFailed: {ep["last_error"]}') + + elif ep["status"] == "LoadSuccessful": + if ep["version"] >= version: + logger.info("LoadSuccessful") + break + else: + logger.info("LoadSuccessful but wrong version") + + if time.time() - start > 10: + raise RuntimeError("Waited more then 10s for deployment") + + logger.info(f"Sleeping {interval}...") + time.sleep(interval) + + def set_credentials(self, username, password): + """ + Set credentials for all the TabPy client-server communication + where client is tabpy-tools and server is tabpy-server. + + Parameters + ---------- + username : str + User name (login). Username is case insensitive. + + password : str + Password in plain text. + """ + self._service.set_credentials(username, password) diff --git a/tabpy/tabpy_tools/custom_query_object.py b/tabpy/tabpy_tools/custom_query_object.py new file mode 100644 index 00000000..18a149b8 --- /dev/null +++ b/tabpy/tabpy_tools/custom_query_object.py @@ -0,0 +1,83 @@ +import logging +from .query_object import QueryObject as _QueryObject + + +logger = logging.getLogger(__name__) + + +class CustomQueryObject(_QueryObject): + def __init__(self, query, description=""): + """Create a new CustomQueryObject. + + Parameters + ----------- + + query : function + Function that defines a custom query method. The query can have any + signature, but input and output of the query needs to be JSON + serializable. + + description : str + The description of the custom query object + + """ + super().__init__(description) + + self.custom_query = query + + def query(self, *args, **kwargs): + """Query the custom defined query method using the given input. + + Parameters + ---------- + args : list + positional arguments to the query + + kwargs : dict + keyword arguments to the query + + Returns + ------- + out: object. + The results depends on the implementation of the query method. + Typically the return value will be whatever that function returns. + + See Also + -------- + QueryObject + """ + # include the dependent files in sys path so that the query can run + # correctly + + try: + logger.debug( + "Running custom query with arguments " f"({args}, {kwargs})..." + ) + ret = self.custom_query(*args, **kwargs) + except Exception as e: + logger.exception( + "Exception hit when running custom query, error: " f"{str(e)}" + ) + raise + + logger.debug(f"Received response {ret}") + try: + return self._make_serializable(ret) + except Exception as e: + logger.exception( + "Cannot properly serialize custom query result, " f"error: {str(e)}" + ) + raise + + def get_doc_string(self): + """Get doc string from customized query""" + if self.custom_query.__doc__ is not None: + return self.custom_query.__doc__ + else: + return "-- no docstring found in query function --" + + def get_methods(self): + return [self.get_query_method()] + + def get_query_method(self): + return {"method": "query"} diff --git a/tabpy/tabpy_tools/query_object.py b/tabpy/tabpy_tools/query_object.py new file mode 100644 index 00000000..5ccbc109 --- /dev/null +++ b/tabpy/tabpy_tools/query_object.py @@ -0,0 +1,108 @@ +import abc +import logging +import os +import json +import shutil + +import cloudpickle as _cloudpickle + + +logger = logging.getLogger(__name__) + + +class QueryObject(abc.ABC): + """ + Derived class needs to implement the following interface: + * query() -- given input, return query result + * get_doc_string() -- returns documentation for the Query Object + """ + + def __init__(self, description=""): + self.description = description + + def get_dependencies(self): + """All endpoints this endpoint depends on""" + return [] + + @abc.abstractmethod + def query(self, input): + """execute query on the provided input""" + pass + + @abc.abstractmethod + def get_doc_string(self): + """Returns documentation for the query object + + By default, this method returns the docstring for 'query' method + Derived class may overwrite this method to dynamically create docstring + """ + pass + + def save(self, path): + """ Save query object to the given local path + + Parameters + ---------- + path : str + The location to save the query object to + """ + if os.path.exists(path): + logger.warning( + f'Overwriting existing file "{path}" when saving query object' + ) + rm_fn = os.remove if os.path.isfile(path) else shutil.rmtree + rm_fn(path) + self._save_local(path) + + def _save_local(self, path): + """Save current query object to local path + """ + try: + os.makedirs(path) + except OSError as e: + import errno + + if e.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + + with open(os.path.join(path, "pickle_archive"), "wb") as f: + _cloudpickle.dump(self, f) + + @classmethod + def load(cls, path): + """ Load query object from given path + """ + new_po = None + new_po = cls._load_local(path) + + logger.info(f'Loaded query object "{type(new_po).__name__}" successfully') + + return new_po + + @classmethod + def _load_local(cls, path): + path = os.path.abspath(os.path.expanduser(path)) + with open(os.path.join(path, "pickle_archive"), "rb") as f: + return _cloudpickle.load(f) + + @classmethod + def _make_serializable(cls, result): + """Convert a result from object query to python data structure that can + easily serialize over network + """ + try: + json.dumps(result) + except TypeError: + raise TypeError( + "Result from object query is not json serializable: " f"{result}" + ) + + return result + + # Returns an array of dictionary that contains the methods and their + # corresponding schema information. + @abc.abstractmethod + def get_methods(self): + return None diff --git a/tabpy/tabpy_tools/rest.py b/tabpy/tabpy_tools/rest.py new file mode 100644 index 00000000..f200c250 --- /dev/null +++ b/tabpy/tabpy_tools/rest.py @@ -0,0 +1,423 @@ +import abc +from collections.abc import MutableMapping +import logging +import requests +from requests.auth import HTTPBasicAuth +from re import compile +import json as json + + +logger = logging.getLogger(__name__) + + +class ResponseError(Exception): + """Raised when we get an unexpected response.""" + + def __init__(self, response): + super().__init__("Unexpected server response") + self.response = response + self.status_code = response.status_code + + try: + r = response.json() + self.info = r["info"] + self.message = response.json()["message"] + except (json.JSONDecodeError, KeyError): + self.info = None + self.message = response.text + + def __str__(self): + return f"({self.status_code}) " f"{self.message} " f"{self.info}" + + +class RequestsNetworkWrapper: + """The NetworkWrapper wraps the underlying network connection to simplify + the interface a bit. This can be replaced with something that can be built + on some other type of network connection, such as PyCURL. + + This version requires you to instantiate a requests session object to your + liking. It will create a generic session for you if you don't specify it, + which you can modify later. + + For authentication, use:: + + session.auth = (username, password) + """ + + def __init__(self, session=None): + # Set .auth as appropriate. + if session is None: + session = requests.session() + + self.session = session + self.auth = None + + @staticmethod + def raise_error(response): + logger.error( + f"Error with server response. code={response.status_code}; " + f"text={response.text}" + ) + + raise ResponseError(response) + + @staticmethod + def _remove_nones(data): + if isinstance(data, dict): + for k in [k for k, v in data.items() if v is None]: + del data[k] + + def _encode_request(self, data): + self._remove_nones(data) + + if data is not None: + return json.dumps(data) + else: + return None + + def GET(self, url, data, timeout=None): + """Issues a GET request to the URL with the data specified. Returns an + object that is parsed from the response JSON.""" + self._remove_nones(data) + + logger.info(f"GET {url} with {data}") + + response = self.session.get(url, params=data, timeout=timeout, auth=self.auth) + if response.status_code != 200: + self.raise_error(response) + logger.info(f"response={response.text}") + + if response.text == "": + return dict() + else: + return response.json() + + def POST(self, url, data, timeout=None): + """Issues a POST request to the URL with the data specified. Returns an + object that is parsed from the response JSON.""" + data = self._encode_request(data) + + logger.info(f"POST {url} with {data}") + response = self.session.post( + url, + data=data, + headers={"content-type": "application/json"}, + timeout=timeout, + auth=self.auth, + ) + + if response.status_code not in (200, 201): + self.raise_error(response) + + return response.json() + + def PUT(self, url, data, timeout=None): + """Issues a PUT request to the URL with the data specified. Returns an + object that is parsed from the response JSON.""" + data = self._encode_request(data) + + logger.info(f"PUT {url} with {data}") + + response = self.session.put( + url, + data=data, + headers={"content-type": "application/json"}, + timeout=timeout, + auth=self.auth, + ) + if response.status_code != 200: + self.raise_error(response) + + return response.json() + + def DELETE(self, url, data, timeout=None): + """ + Issues a DELETE request to the URL with the data specified. Returns an + object that is parsed from the response JSON. + """ + if data is not None: + data = json.dumps(data) + + logger.info(f"DELETE {url} with {data}") + + response = self.session.delete(url, data=data, timeout=timeout, auth=self.auth) + + if response.status_code <= 499 and response.status_code >= 400: + raise RuntimeError(response.text) + + if response.status_code not in (200, 201, 204): + raise RuntimeError( + f"Error with server response code: {response.status_code}" + ) + + def set_credentials(self, username, password): + """ + Set credentials for all the TabPy client-server communication + where client is tabpy-tools and server is tabpy-server. + + Parameters + ---------- + username : str + User name (login). Username is case insensitive. + + password : str + Password in plain text. + """ + logger.info(f"Setting credentials (username: {username})") + self.auth = HTTPBasicAuth(username, password) + + +class ServiceClient: + """ + A generic service client. + + This will take an endpoint URL and a network_wrapper. You can use the + RequestsNetworkWrapper if you want to use the requests module. The + endpoint URL is prepended to all the requests and forwarded to the network + wrapper. + """ + + def __init__(self, endpoint, network_wrapper=None): + if network_wrapper is None: + network_wrapper = RequestsNetworkWrapper(session=requests.session()) + + self.network_wrapper = network_wrapper + + pattern = compile(".*(:[0-9]+)$") + if not endpoint.endswith("/") and not pattern.match(endpoint): + logger.warning(f"endpoint {endpoint} does not end with '/': appending.") + endpoint = endpoint + "/" + + self.endpoint = endpoint + + def GET(self, url, data=None, timeout=None): + """Prepends self.endpoint to the url and issues a GET request.""" + return self.network_wrapper.GET(self.endpoint + url, data, timeout) + + def POST(self, url, data=None, timeout=None): + """Prepends self.endpoint to the url and issues a POST request.""" + return self.network_wrapper.POST(self.endpoint + url, data, timeout) + + def PUT(self, url, data=None, timeout=None): + """Prepends self.endpoint to the url and issues a PUT request.""" + return self.network_wrapper.PUT(self.endpoint + url, data, timeout) + + def DELETE(self, url, data=None, timeout=None): + """Prepends self.endpoint to the url and issues a DELETE request.""" + self.network_wrapper.DELETE(self.endpoint + url, data, timeout) + + def set_credentials(self, username, password): + """ + Set credentials for all the TabPy client-server communication + where client is tabpy-tools and server is tabpy-server. + + Parameters + ---------- + username : str + User name (login). Username is case insensitive. + + password : str + Password in plain text. + """ + self.network_wrapper.set_credentials(username, password) + + +class RESTProperty: + """A descriptor that will control the type of value stored.""" + + def __init__(self, type, from_json=lambda x: x, to_json=lambda x: x, doc=None): + self.__doc__ = doc + self.type = type + self.from_json = from_json + self.to_json = to_json + + def __get__(self, instance, _): + if instance: + try: + return getattr(instance, self.name) + except AttributeError: + raise AttributeError(f"{self.name} has not been set yet.") + else: + return self + + def __set__(self, instance, value): + if value is not None and not isinstance(value, self.type): + value = self.type(value) + + setattr(instance, self.name, value) + + def __delete__(self, instance): + delattr(instance, self.name) + + +class _RESTMetaclass(abc.ABCMeta): + """The metaclass for RESTObjects. + + This will look into the attributes for the class. If they are a + RESTProperty, then it will add it to the __rest__ set and give it its + name. + + If the bases have __rest__, then it will add them to the __rest__ set as + well. + """ + + def __init__(self, name, bases, dict): + super().__init__(name, bases, dict) + + self.__rest__ = set() + for base in bases: + self.__rest__.update(getattr(base, "__rest__", set())) + + for k, v in dict.items(): + if isinstance(v, RESTProperty): + v.__dict__["name"] = "_" + k + self.__rest__.add(k) + + +class RESTObject(MutableMapping, metaclass=_RESTMetaclass): + """A base class that has methods generally useful for interacting with + REST objects. The attributes are accessible either as dict keys or as + attributes. The object also behaves like a dict, even replicating the + repr() functionality. + + Attributes + ---------- + + __rest__ : set of str + A set of all the rest attribute names. This is generated automatically + and should include all of the base classes' __rest__ as well as any + addition RESTProperty. + + """ + + """ __metaclass__ = _RESTMetaclass""" + + def __init__(self, **kwargs): + """Creates a new instance of the RESTObject. + + Parameters + ---------- + + The parameters depend on __rest__. Each item in __rest__ is searched + for. If found, it is assigned to the instance. Additional parameters + are ignored. + + """ + logger.info(f"Initializing {self.__class__.__name__} from {kwargs}") + for attr in self.__rest__: + if attr in kwargs: + setattr(self, attr, kwargs.pop(attr)) + + def __repr__(self): + return ( + "{" + ", ".join([repr(k) + ": " + repr(v) for k, v in self.items()]) + "}" + ) + + @classmethod + def from_json(cls, data): + """Returns a new class object with data populated from json.loads().""" + attrs = {} + for attr in cls.__rest__: + try: + value = data[attr] + except KeyError: + pass + else: + prop = cls.__dict__[attr] + attrs[attr] = prop.from_json(value) + return cls(**attrs) + + def to_json(self): + """Returns a dict representing this object. This dict will be sent to + json.dumps(). + + The keys are the items in __rest__ and the values are the current + values. If missing, it is not included. + """ + result = {} + for attr in self.__rest__: + prop = getattr(self.__class__, attr) + try: + result[attr] = prop.to_json(getattr(self, attr)) + except AttributeError: + pass + + return result + + def __eq__(self, other): + return isinstance(self, type(other)) and all( + (getattr(self, a) == getattr(other, a) for a in self.__rest__) + ) + + def __len__(self): + return len([a for a in self.__rest__ if hasattr(self, "_" + a)]) + + def __iter__(self): + return iter([a for a in self.__rest__ if hasattr(self, "_" + a)]) + + def __getitem__(self, item): + if item not in self.__rest__: + raise KeyError(item) + try: + return getattr(self, item) + except AttributeError: + raise KeyError(item) + + def __setitem__(self, item, value): + if item not in self.__rest__: + raise KeyError(item) + setattr(self, item, value) + + def __delitem__(self, item): + if item not in self.__rest__: + raise KeyError(item) + try: + delattr(self, "_" + item) + except AttributeError: + raise KeyError(item) + + def __contains__(self, item): + return item in self.__rest__ + + +def enum(*values, **kwargs): + """Generates an enum function that only accepts particular values. Other + values will raise a ValueError. + + Parameters + ---------- + + values : list + These are the acceptable values. + + type : type + The acceptable types of values. Values will be converted before being + checked against the allowed values. If not specified, no conversion + will be performed. + + Example + ------- + + >>> my_enum = enum(1, 2, 3, 4, 5, type=int) + >>> a = my_enum(1) + >>> b = my_enum(2) + >>> c = mu_enum(6) # Raises ValueError + + """ + if len(values) < 1: + raise ValueError("At least one value is required.") + enum_type = kwargs.pop("type", str) + if kwargs: + raise TypeError(f'Unexpected parameters: {", ".join(kwargs.keys())}') + + def __new__(cls, value): + if value not in cls.values: + raise ValueError( + f"{value} is an unexpected value. " f"Expected one of {cls.values}" + ) + + return super(enum, cls).__new__(cls, value) + + enum = type("Enum", (enum_type,), {"values": values, "__new__": __new__}) + + return enum diff --git a/tabpy/tabpy_tools/rest_client.py b/tabpy/tabpy_tools/rest_client.py new file mode 100644 index 00000000..eb0ef211 --- /dev/null +++ b/tabpy/tabpy_tools/rest_client.py @@ -0,0 +1,252 @@ +from .rest import RESTObject, RESTProperty +from datetime import datetime + + +def from_epoch(value): + if isinstance(value, datetime): + return value + else: + return datetime.utcfromtimestamp(value) + + +def to_epoch(value): + return (value - datetime(1970, 1, 1)).total_seconds() + + +class Endpoint(RESTObject): + """Represents an endpoint. + + Note that not every attribute is returned as part of the GET. + + Attributes + ---------- + + name : str + The name of the endpoint. Valid names include ``[a-zA-Z0-9_\\- ]+`` + type : str + The type of endpoint. The types include "alias", "model". + version : int + The version of this endpoint. Initial versions have version on 1. New + versions increment this by 1. + description : str + A human-readable description of the endpoint. + dependencies: list + A list of endpoints that this endpoint depends on. + methods : list + ??? + """ + + name = RESTProperty(str) + type = RESTProperty(str) + version = RESTProperty(int) + description = RESTProperty(str) + dependencies = RESTProperty(list) + methods = RESTProperty(list) + creation_time = RESTProperty(datetime, from_epoch, to_epoch) + last_modified_time = RESTProperty(datetime, from_epoch, to_epoch) + evaluator = RESTProperty(str) + schema_version = RESTProperty(int) + schema = RESTProperty(str) + + def __new__(cls, **kwargs): + """Dispatch to the appropriate class.""" + cls = {"alias": AliasEndpoint, "model": ModelEndpoint}[kwargs["type"]] + + """return object.__new__(cls, **kwargs)""" + """ modified for Python 3""" + return object.__new__(cls) + + def __eq__(self, other): + return ( + self.name == other.name + and self.type == other.type + and self.version == other.version + and self.description == other.description + and self.dependencies == other.dependencies + and self.methods == other.methods + and self.evaluator == other.evaluator + and self.schema_version == other.schema_version + and self.schema == other.schema + ) + + +class ModelEndpoint(Endpoint): + """Represents a model endpoint. + + src_path : str + + The local file path to the source of this object. + + required_files : str + + The local file path to the directory containing the + required files. + + required_packages : str + + The local file path to the directory containing the + required packages. + + """ + + src_path = RESTProperty(str) + required_files = RESTProperty(list) + required_packages = RESTProperty(list) + required_packages_dst_path = RESTProperty(str) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.type = "model" + + def __eq__(self, other): + return ( + super().__eq__(other) + and self.required_files == other.required_files + and self.required_packages == other.required_packages + ) + + +class AliasEndpoint(Endpoint): + """Represents an alias Endpoint. + + target : str + + The endpoint that this is an alias for. + + """ + + target = RESTProperty(str) + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self.type = "alias" + + +class RESTServiceClient: + """A thin client for the REST Service.""" + + def __init__(self, service_client): + self.service_client = service_client + self.query_timeout = None + + def get_info(self): + """Returns the /info""" + return self.service_client.GET("info") + + def query(self, name, *args, **kwargs): + """Performs a query. Either specify *args or **kwargs, not both. + Respects query_timeout.""" + if args and kwargs: + raise ValueError( + "Mixing of keyword arguments and positional arguments when " + "querying an endpoint is not supported." + ) + return self.service_client.POST( + "query/" + name, data={"data": args or kwargs}, timeout=self.query_timeout + ) + + def get_endpoint_upload_destination(self): + """Returns a dict representing where endpoint data should be uploaded. + + Returns + ------- + dict + Keys include: + * path: a local file path. + + Note: In the future, other paths and parameters may be supported. + + Note: At this time, the response should not change over time. + """ + return self.service_client.GET("configurations/endpoint_upload_destination") + + def get_endpoints(self, type=None): + """Returns endpoints from the management API. + + Parameters + ---------- + + type : str + The type of endpoint to return. None will include all endpoints. + Other options are 'model' and 'alias'. + """ + result = {} + for name, attrs in self.service_client.GET("endpoints", {"type": type}).items(): + endpoint = Endpoint.from_json(attrs) + endpoint.name = name + result[name] = endpoint + return result + + def get_endpoint(self, endpoint_name): + """Returns an endpoints from the management API given its name. + + Parameters + ---------- + + endpoint_name : str + + The name of the endpoint. + """ + ((name, attrs),) = self.service_client.GET("endpoints/" + endpoint_name).items() + endpoint = Endpoint.from_json(attrs) + endpoint.name = name + return endpoint + + def add_endpoint(self, endpoint): + """Adds an endpoint through the management API. + + Parameters + ---------- + + endpoint : Endpoint + """ + return self.service_client.POST("endpoints", endpoint.to_json()) + + def set_endpoint(self, endpoint): + """Updates an endpoint through the management API. + + Parameters + ---------- + + endpoint : Endpoint + + The endpoint to update. + """ + return self.service_client.PUT("endpoints/" + endpoint.name, endpoint.to_json()) + + def remove_endpoint(self, endpoint_name): + """Deletes an endpoint through the management API. + + Parameters + ---------- + + endpoint_name : str + + The endpoint to delete. + """ + self.service_client.DELETE("endpoints/" + endpoint_name) + + def get_status(self): + """Returns the status of the server. + + Returns + ------- + + dict + """ + return self.service_client.GET("status") + + def set_credentials(self, username, password): + """ + Set credentials for all the TabPy client-server communication + where client is tabpy-tools and server is tabpy-server. + + Parameters + ---------- + username : str + User name (login). Username is case insensitive. + + password : str + Password in plain text. + """ + self.service_client.set_credentials(username, password) diff --git a/tabpy/tabpy_tools/schema.py b/tabpy/tabpy_tools/schema.py new file mode 100644 index 00000000..8bd0ab9a --- /dev/null +++ b/tabpy/tabpy_tools/schema.py @@ -0,0 +1,108 @@ +import logging +import genson +import jsonschema + + +logger = logging.getLogger(__name__) + + +def _generate_schema_from_example_and_description(input, description): + """ + With an example input, a schema is automatically generated that conforms + to the example in json-schema.org. The description given by the users + is then added to the schema. + """ + s = genson.SchemaBuilder(None) + s.add_object(input) + input_schema = s.to_schema() + + if description is not None: + if "properties" in input_schema: + # Case for input = {'x':1}, input_description='not a dict' + if not isinstance(description, dict): + msg = f"{input} and {description} do not match" + logger.error(msg) + raise Exception(msg) + + for key in description: + # Case for input = {'x':1}, + # input_description={'x':'x value', 'y':'y value'} + if key not in input_schema["properties"]: + msg = f"{key} not found in {input}" + logger.error(msg) + raise Exception(msg) + else: + input_schema["properties"][key]["description"] = description[key] + else: + if isinstance(description, dict): + raise Exception(f"{input} and {description} do not match") + else: + input_schema["description"] = description + + try: + # This should not fail unless there are bugs with either genson or + # jsonschema. + jsonschema.validate(input, input_schema) + except Exception as e: + logger.error(f"Internal error validating schema: {str(e)}") + raise + + return input_schema + + +def generate_schema(input, output, input_description=None, output_description=None): + """ + Generate schema from a given sample input and output. + A generated schema can be passed to a server together with a function to + annotate it with information about input and output parameters, and + examples thereof. The schema needs to follow the conventions of JSON Schema + (see json-schema.org). + + Parameters + ----------- + input : any python type | dict + output: any python type | dict + input_description : str | dict, optional + output_description : str | dict, optional + + References + ----------- + - `Json Schema ` + + Examples + ---------- + .. sourcecode:: python + For just one input parameter, state the example directly. + >>> from tabpy.tabpy_tools.schema import generate_schema + >>> schema = generate_schema( + input=5, + output=25, + input_description='input value', + output_description='the squared value of input') + >>> schema + {'sample': 5, + 'input': {'type': 'integer', 'description': 'input value'}, + 'output': {'type': 'integer', 'description': 'the squared value of input'}} + For two or more input parameters, specify them using a dictionary. + >>> import graphlab + >>> schema = generate_schema( + input={'x': 3, 'y': 2}, + output=6, + input_description={'x': 'value of x', + 'y': 'value of y'}, + output_description='x times y') + >>> schema + {'sample': {'y': 2, 'x': 3}, + 'input': {'required': ['x', 'y'], + 'type': 'object', + 'properties': {'y': {'type': 'integer', 'description': 'value of y'}, + 'x': {'type': 'integer', 'description': 'value of x'}}}, + 'output': {'type': 'integer', 'description': 'x times y'}} + """ # noqa: E501 + input_schema = _generate_schema_from_example_and_description( + input, input_description + ) + output_schema = _generate_schema_from_example_and_description( + output, output_description + ) + return {"input": input_schema, "sample": input, "output": output_schema} From a647292afb5e424cbda39fc1db43f038bf67ba92 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Mon, 27 Jul 2020 16:14:03 -0700 Subject: [PATCH 45/61] Unit and integration tests passing --- tabpy/tabpy_server/__init__.py | 0 tabpy/tabpy_server/app/app.py | 6 +++--- tests/integration/test_deploy_and_evaluate_model.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) delete mode 100644 tabpy/tabpy_server/__init__.py diff --git a/tabpy/tabpy_server/__init__.py b/tabpy/tabpy_server/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 735218ca..3a24b12a 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -266,9 +266,9 @@ def _parse_config(self, config_file): None, None), (SettingsParameters.KeyFile, ConfigParameters.TABPY_KEY_FILE, None, None), (SettingsParameters.StateFilePath, ConfigParameters.TABPY_STATE_PATH, - os.path.join(pkg_path, "server"), None), + os.path.join(pkg_path, "tabpy_server"), None), (SettingsParameters.StaticPath, ConfigParameters.TABPY_STATIC_PATH, - os.path.join(pkg_path, "server", "static"), None), + os.path.join(pkg_path, "tabpy_server", "static"), None), (ConfigParameters.TABPY_PWD_FILE, ConfigParameters.TABPY_PWD_FILE, None, None), (SettingsParameters.LogRequestContext, ConfigParameters.TABPY_LOG_DETAILS, "false", None), @@ -417,7 +417,7 @@ def _build_tabpy_state(self): state_file_path = os.path.join(state_file_dir, "state.ini") if not os.path.isfile(state_file_path): state_file_template_path = os.path.join( - pkg_path, "server", "state.ini.template" + pkg_path, "tabpy_server", "state.ini.template" ) logger.debug( f"File {state_file_path} not found, creating from " diff --git a/tests/integration/test_deploy_and_evaluate_model.py b/tests/integration/test_deploy_and_evaluate_model.py index 1fa2fd80..91b071a5 100644 --- a/tests/integration/test_deploy_and_evaluate_model.py +++ b/tests/integration/test_deploy_and_evaluate_model.py @@ -12,7 +12,7 @@ def test_deploy_and_evaluate_model(self): # Uncomment the following line to preserve # test case output and other files (config, state, ect.) # in system temp folder. - self.set_delete_temp_folder(False) + # self.set_delete_temp_folder(False) self.deploy_models(self._get_username(), self._get_password()) From 55e36b29751c6c220daefad8bab6ea6b3dbac1c0 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Tue, 28 Jul 2020 15:02:40 -0700 Subject: [PATCH 46/61] Update .gitignore --- .gitignore | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 1ebc2208..122b5066 100644 --- a/.gitignore +++ b/.gitignore @@ -117,9 +117,9 @@ package-lock.json .idea/ # TabPy server artifacts -tabpy/server/state.ini -tabpy/server/query_objects -tabpy/server/staging +tabpy/tabpy_server/state.ini +tabpy/tabpy_server/query_objects +tabpy/tabpy_server/staging # VS Code *.code-workspace From 012e8020df987a9eba18ac141ad3156a3b67bf29 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Tue, 28 Jul 2020 15:11:20 -0700 Subject: [PATCH 47/61] do not track settings.json for VSCode --- .gitignore | 2 +- .vscode/settings.json | 23 ----------------------- 2 files changed, 1 insertion(+), 24 deletions(-) delete mode 100755 .vscode/settings.json diff --git a/.gitignore b/.gitignore index 122b5066..614ac726 100644 --- a/.gitignore +++ b/.gitignore @@ -123,7 +123,7 @@ tabpy/tabpy_server/staging # VS Code *.code-workspace -.vscode +.vscode/ # etc setup.bat diff --git a/.vscode/settings.json b/.vscode/settings.json deleted file mode 100755 index 2b4fa02e..00000000 --- a/.vscode/settings.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - "git.enabled": true, - "files.exclude": { - "**/build": true, - "**/dist": true, - "**/__pycache__": true, - "**/.pytest_cache": true, - "**/*.egg-info": true, - "**/*.pyc": true - }, - "python.linting.pylintEnabled": false, - "python.linting.flake8Enabled": true, - "python.linting.enabled": true, - "python.testing.autoTestDiscoverOnSaveEnabled": true, - "python.testing.pytestArgs": [ - "." - ], - "python.testing.unittestEnabled": false, - "python.testing.nosetestsEnabled": false, - "python.testing.pytestEnabled": true, - "python.linting.pycodestyleEnabled": false, - "python.pythonPath": "C:\\Users\\ogolovatyi\\Anaconda3\\envs\\Python 37\\python.exe" -} \ No newline at end of file From d1e54b9ea4fcecc21d735b45ffad655036a7afe8 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Tue, 28 Jul 2020 15:15:06 -0700 Subject: [PATCH 48/61] Fix server -> tabpy_server --- MANIFEST.in | 8 ++++---- docs/TableauConfiguration.md | 6 +++--- docs/server-config.md | 6 +++--- tabpy/tabpy_server/common/default.conf | 2 +- .../integration/resources/deploy_and_evaluate_model.conf | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 55289a56..2d93e579 100755 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,10 +1,10 @@ exclude \ - tabpy/server/state.ini + tabpy/tabpy_server/state.ini include \ CHANGELOG \ LICENSE \ tabpy/VERSION \ - tabpy/server/state.ini.template \ - tabpy/server/static/* \ - tabpy/server/common/default.conf + tabpy/tabpy_server/state.ini.template \ + tabpy/tabpy_server/static/* \ + tabpy/tabpy_server/common/default.conf diff --git a/docs/TableauConfiguration.md b/docs/TableauConfiguration.md index 6fb5fadd..74ced168 100755 --- a/docs/TableauConfiguration.md +++ b/docs/TableauConfiguration.md @@ -29,12 +29,12 @@ documentation page. To configure Tableau Server 2018.2 and newer versions to connect to TabPy server follow instructions on Tableau -[Configure Connections to Analytics Extensions](https://onlinehelp.tableau.com/current/server/en-us/tsm.htm) +[Configure Connections to Analytics Extensions](https://onlinehelp.tableau.com/current/tabpy_server/en-us/tsm.htm) page. Specific details about how to configure a secure connection to TabPy, enable or disable connections and other setting can be found at Tableau -[TSM Security documentation](https://onlinehelp.tableau.com/current/server/en-us/cli_security_tsm.htm#tsm_security_vizql-extsvc-ssl-enable) +[TSM Security documentation](https://onlinehelp.tableau.com/current/tabpy_server/en-us/cli_security_tsm.htm#tsm_security_vizql-extsvc-ssl-enable) page. For how to configure TabPy instance follow instructions at @@ -44,7 +44,7 @@ For how to configure TabPy instance follow instructions at For Tableau workbooks with embedded Python code to work on Tableau Server 10.1 or later, you need to go through a similar setup but using the -[tabadmin](https://onlinehelp.tableau.com/current/server/en-us/tabadmin.htm) +[tabadmin](https://onlinehelp.tableau.com/current/tabpy_server/en-us/tabadmin.htm) command line utility. The two server settings that need to be configured are `vizqlserver.extsvc.host` and `vizqlserver.extsvc.port`. diff --git a/docs/server-config.md b/docs/server-config.md index 543304da..6b4cda0d 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -65,7 +65,7 @@ at [`logging.config` documentation page](https://docs.python.org/3.6/library/log server. Default value - `tabpy/server` subfolder in TabPy package folder. - `TABPY_STATIC_PATH` - absolute path for location of static files (index.html - page) for TabPy instance. Default value - `tabpy/server/static` + page) for TabPy instance. Default value - `tabpy/tabpy_server/static` subfolder in TabPy package folder. - `TABPY_PWD_FILE` - absolute path to password file. Setting up this parameter makes TabPy require credentials with HTTP(S) requests. More details about @@ -104,7 +104,7 @@ settings._ # TABPY_STATE_PATH = /tabpy/server # Where static pages live -# TABPY_STATIC_PATH = /tabpy/server/static +# TABPY_STATIC_PATH = /tabpy/tabpy_server/static # For how to configure TabPy authentication read # docs/server-config.md. @@ -256,7 +256,7 @@ as explained in Python documentation at [Logging Configuration page](https://docs.python.org/3.6/library/logging.config.html). A default config provided with TabPy is at -[`tabpy-server/server/common/default.conf`](tabpy-server/server/common/default.conf) +[`tabpy-server/tabpy_server/common/default.conf`](tabpy-server/tabpy_server/common/default.conf) and has a configuration for console and file loggers. Changing the config file allows the user to modify the log level, format of the logged messages and add or remove loggers. diff --git a/tabpy/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf index 80e9dd49..e964e80c 100644 --- a/tabpy/tabpy_server/common/default.conf +++ b/tabpy/tabpy_server/common/default.conf @@ -4,7 +4,7 @@ # TABPY_STATE_PATH = ./tabpy/server # Where static pages live -# TABPY_STATIC_PATH = ./tabpy/server/static +# TABPY_STATIC_PATH = ./tabpy/tabpy_server/static # For how to configure TabPy authentication read # Authentication section in docs/server-config.md. diff --git a/tests/integration/resources/deploy_and_evaluate_model.conf b/tests/integration/resources/deploy_and_evaluate_model.conf index 4b1ab753..684fe27b 100755 --- a/tests/integration/resources/deploy_and_evaluate_model.conf +++ b/tests/integration/resources/deploy_and_evaluate_model.conf @@ -4,7 +4,7 @@ TABPY_PORT = 9008 # TABPY_STATE_PATH = ./tabpy/server # Where static pages live -# TABPY_STATIC_PATH = ./tabpy/server/static +# TABPY_STATIC_PATH = ./tabpy/tabpy_server/static # For how to configure TabPy authentication read # Authentication section in docs/server-config.md. From 8fdf6b0e85b8e167802473646b76b841760f38ab Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Tue, 28 Jul 2020 15:24:36 -0700 Subject: [PATCH 49/61] more cleaning for tabpy_server names --- docs/TableauConfiguration.md | 6 +++--- docs/server-config.md | 4 ++-- setup.py | 6 +++--- tabpy/tabpy_server/common/default.conf | 2 +- tests/integration/resources/deploy_and_evaluate_model.conf | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/TableauConfiguration.md b/docs/TableauConfiguration.md index 74ced168..6fb5fadd 100755 --- a/docs/TableauConfiguration.md +++ b/docs/TableauConfiguration.md @@ -29,12 +29,12 @@ documentation page. To configure Tableau Server 2018.2 and newer versions to connect to TabPy server follow instructions on Tableau -[Configure Connections to Analytics Extensions](https://onlinehelp.tableau.com/current/tabpy_server/en-us/tsm.htm) +[Configure Connections to Analytics Extensions](https://onlinehelp.tableau.com/current/server/en-us/tsm.htm) page. Specific details about how to configure a secure connection to TabPy, enable or disable connections and other setting can be found at Tableau -[TSM Security documentation](https://onlinehelp.tableau.com/current/tabpy_server/en-us/cli_security_tsm.htm#tsm_security_vizql-extsvc-ssl-enable) +[TSM Security documentation](https://onlinehelp.tableau.com/current/server/en-us/cli_security_tsm.htm#tsm_security_vizql-extsvc-ssl-enable) page. For how to configure TabPy instance follow instructions at @@ -44,7 +44,7 @@ For how to configure TabPy instance follow instructions at For Tableau workbooks with embedded Python code to work on Tableau Server 10.1 or later, you need to go through a similar setup but using the -[tabadmin](https://onlinehelp.tableau.com/current/tabpy_server/en-us/tabadmin.htm) +[tabadmin](https://onlinehelp.tableau.com/current/server/en-us/tabadmin.htm) command line utility. The two server settings that need to be configured are `vizqlserver.extsvc.host` and `vizqlserver.extsvc.port`. diff --git a/docs/server-config.md b/docs/server-config.md index 6b4cda0d..28fa0deb 100755 --- a/docs/server-config.md +++ b/docs/server-config.md @@ -62,7 +62,7 @@ at [`logging.config` documentation page](https://docs.python.org/3.6/library/log [TabPy Tools documentation](tabpy-tools.md) for details. Default value - `/tmp/query_objects`. - `TABPY_STATE_PATH` - state folder location (absolute path) for Tornado web - server. Default value - `tabpy/server` subfolder in TabPy package + server. Default value - `tabpy/tabpy_server` subfolder in TabPy package folder. - `TABPY_STATIC_PATH` - absolute path for location of static files (index.html page) for TabPy instance. Default value - `tabpy/tabpy_server/static` @@ -101,7 +101,7 @@ settings._ [TabPy] # TABPY_QUERY_OBJECT_PATH = /tmp/query_objects # TABPY_PORT = 9004 -# TABPY_STATE_PATH = /tabpy/server +# TABPY_STATE_PATH = /tabpy/tabpy_server # Where static pages live # TABPY_STATIC_PATH = /tabpy/tabpy_server/static diff --git a/setup.py b/setup.py index 11828310..d4fd80bd 100755 --- a/setup.py +++ b/setup.py @@ -54,9 +54,9 @@ def read(fname): package_data={ "tabpy": [ "VERSION", - "server/state.ini.template", - "server/static/*", - "server/common/default.conf", + "tabpy_server/state.ini.template", + "tabpy_server/static/*", + "tabpy_server/common/default.conf", ] }, python_requires=">=3.6", diff --git a/tabpy/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf index e964e80c..ee02453d 100644 --- a/tabpy/tabpy_server/common/default.conf +++ b/tabpy/tabpy_server/common/default.conf @@ -1,7 +1,7 @@ [TabPy] # TABPY_QUERY_OBJECT_PATH = /tmp/query_objects # TABPY_PORT = 9004 -# TABPY_STATE_PATH = ./tabpy/server +# TABPY_STATE_PATH = ./tabpy/tabpy_server # Where static pages live # TABPY_STATIC_PATH = ./tabpy/tabpy_server/static diff --git a/tests/integration/resources/deploy_and_evaluate_model.conf b/tests/integration/resources/deploy_and_evaluate_model.conf index 684fe27b..375ea47a 100755 --- a/tests/integration/resources/deploy_and_evaluate_model.conf +++ b/tests/integration/resources/deploy_and_evaluate_model.conf @@ -1,7 +1,7 @@ [TabPy] # TABPY_QUERY_OBJECT_PATH = /tmp/query_objects TABPY_PORT = 9008 -# TABPY_STATE_PATH = ./tabpy/server +# TABPY_STATE_PATH = ./tabpy/tabpy_server # Where static pages live # TABPY_STATIC_PATH = ./tabpy/tabpy_server/static From d966774c1ba73e611e49543aca4739a13b7f9235 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Tue, 28 Jul 2020 17:47:30 -0700 Subject: [PATCH 50/61] make "python setup.py test" work --- setup.py | 3 ++- tabpy/VERSION | 2 +- tabpy/tabpy_server/app/app.py | 17 ++++++++++++----- tests/__init__.py | 0 tests/integration/__init__.py | 0 tests/integration/test_auth.py | 2 +- .../integration/test_custom_evaluate_timeout.py | 2 +- .../test_deploy_and_evaluate_model.py | 2 +- .../test_deploy_and_evaluate_model_ssl.py | 2 +- .../test_deploy_model_ssl_off_auth_off.py | 2 +- .../test_deploy_model_ssl_off_auth_on.py | 2 +- .../test_deploy_model_ssl_on_auth_off.py | 2 +- .../test_deploy_model_ssl_on_auth_on.py | 2 +- tests/integration/test_evaluate.py | 2 +- tests/integration/test_url.py | 2 +- tests/integration/test_url_ssl.py | 2 +- tests/unit/__init__.py | 0 tests/unit/server_tests/__init__.py | 0 18 files changed, 26 insertions(+), 18 deletions(-) create mode 100644 tests/__init__.py create mode 100644 tests/integration/__init__.py create mode 100644 tests/unit/__init__.py create mode 100644 tests/unit/server_tests/__init__.py diff --git a/setup.py b/setup.py index d4fd80bd..4d5c8562 100755 --- a/setup.py +++ b/setup.py @@ -7,6 +7,7 @@ import os from setuptools import setup, find_packages +import unittest DOCLINES = (__doc__ or "").split("\n") @@ -106,7 +107,7 @@ def read(fname): "sklearn", "textblob", ], - test_suite="pytest", + test_suite="tests", ) diff --git a/tabpy/VERSION b/tabpy/VERSION index 26aaba0e..867e5243 100755 --- a/tabpy/VERSION +++ b/tabpy/VERSION @@ -1 +1 @@ -1.2.0 +1.2.0 \ No newline at end of file diff --git a/tabpy/tabpy_server/app/app.py b/tabpy/tabpy_server/app/app.py index 3a24b12a..cf2c8ae6 100644 --- a/tabpy/tabpy_server/app/app.py +++ b/tabpy/tabpy_server/app/app.py @@ -60,7 +60,7 @@ class TabPyApp: python_service = None credentials = {} - def __init__(self, config_file=None): + def __init__(self, config_file): if config_file is None: config_file = os.path.join( os.path.dirname(__file__), os.path.pardir, "common", "default.conf" @@ -243,13 +243,20 @@ def _parse_config(self, config_file): pkg_path = os.path.dirname(tabpy.__file__) parser = configparser.ConfigParser(os.environ) + logger.info(f"Parsing config file {config_file}") + file_exists = False if os.path.isfile(config_file): - with open(config_file) as f: - parser.read_string(f.read()) - else: + try: + with open(config_file, 'r') as f: + parser.read_string(f.read()) + file_exists = True + except Exception: + pass + + if not file_exists: logger.warning( - f"Unable to find config file at {config_file}, " + f"Unable to open config file {config_file}, " "using default settings." ) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/test_auth.py b/tests/integration/test_auth.py index 4b1a4bca..e1278fc1 100755 --- a/tests/integration/test_auth.py +++ b/tests/integration/test_auth.py @@ -1,5 +1,5 @@ import base64 -import integ_test_base +from . import integ_test_base class TestAuth(integ_test_base.IntegTestBase): diff --git a/tests/integration/test_custom_evaluate_timeout.py b/tests/integration/test_custom_evaluate_timeout.py index 99bec14c..b9ea1ced 100644 --- a/tests/integration/test_custom_evaluate_timeout.py +++ b/tests/integration/test_custom_evaluate_timeout.py @@ -1,4 +1,4 @@ -import integ_test_base +from . import integ_test_base class TestCustomEvaluateTimeout(integ_test_base.IntegTestBase): diff --git a/tests/integration/test_deploy_and_evaluate_model.py b/tests/integration/test_deploy_and_evaluate_model.py index 91b071a5..29a5cd02 100644 --- a/tests/integration/test_deploy_and_evaluate_model.py +++ b/tests/integration/test_deploy_and_evaluate_model.py @@ -1,4 +1,4 @@ -import integ_test_base +from . import integ_test_base class TestDeployAndEvaluateModel(integ_test_base.IntegTestBase): diff --git a/tests/integration/test_deploy_and_evaluate_model_ssl.py b/tests/integration/test_deploy_and_evaluate_model_ssl.py index 2de1c350..b4e643c7 100755 --- a/tests/integration/test_deploy_and_evaluate_model_ssl.py +++ b/tests/integration/test_deploy_and_evaluate_model_ssl.py @@ -1,4 +1,4 @@ -import integ_test_base +from . import integ_test_base import requests diff --git a/tests/integration/test_deploy_model_ssl_off_auth_off.py b/tests/integration/test_deploy_model_ssl_off_auth_off.py index b754ccfc..34ee3671 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_off.py @@ -1,4 +1,4 @@ -import integ_test_base +from . import integ_test_base class TestDeployModelSSLOffAuthOff(integ_test_base.IntegTestBase): diff --git a/tests/integration/test_deploy_model_ssl_off_auth_on.py b/tests/integration/test_deploy_model_ssl_off_auth_on.py index 09e5c122..b7666acb 100644 --- a/tests/integration/test_deploy_model_ssl_off_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_off_auth_on.py @@ -1,4 +1,4 @@ -import integ_test_base +from . import integ_test_base import base64 diff --git a/tests/integration/test_deploy_model_ssl_on_auth_off.py b/tests/integration/test_deploy_model_ssl_on_auth_off.py index f7a81709..5a03c236 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_off.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_off.py @@ -1,4 +1,4 @@ -import integ_test_base +from . import integ_test_base import requests diff --git a/tests/integration/test_deploy_model_ssl_on_auth_on.py b/tests/integration/test_deploy_model_ssl_on_auth_on.py index 6e833162..af141b72 100644 --- a/tests/integration/test_deploy_model_ssl_on_auth_on.py +++ b/tests/integration/test_deploy_model_ssl_on_auth_on.py @@ -1,4 +1,4 @@ -import integ_test_base +from . import integ_test_base import base64 import requests diff --git a/tests/integration/test_evaluate.py b/tests/integration/test_evaluate.py index ade8600a..69502d41 100755 --- a/tests/integration/test_evaluate.py +++ b/tests/integration/test_evaluate.py @@ -2,7 +2,7 @@ Script evaluation tests. """ -import integ_test_base +from . import integ_test_base import json diff --git a/tests/integration/test_url.py b/tests/integration/test_url.py index 0fa4c03c..d2744b42 100755 --- a/tests/integration/test_url.py +++ b/tests/integration/test_url.py @@ -2,7 +2,7 @@ All other misc. URL-related integration tests. """ -import integ_test_base +from . import integ_test_base class TestURL(integ_test_base.IntegTestBase): diff --git a/tests/integration/test_url_ssl.py b/tests/integration/test_url_ssl.py index 4ded4d5c..8eed8b2c 100755 --- a/tests/integration/test_url_ssl.py +++ b/tests/integration/test_url_ssl.py @@ -3,7 +3,7 @@ when SSL is turned on for TabPy. """ -import integ_test_base +from . import integ_test_base import requests diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/server_tests/__init__.py b/tests/unit/server_tests/__init__.py new file mode 100644 index 00000000..e69de29b From 3c7922ff2d0bd1109b51ca900dfdcf05526c2136 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Tue, 28 Jul 2020 17:53:20 -0700 Subject: [PATCH 51/61] add coverage module as required --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 4d5c8562..644e5dff 100755 --- a/setup.py +++ b/setup.py @@ -98,11 +98,15 @@ def read(fname): }, setup_requires=["pytest-runner"], tests_require=[ + "coverage", + "coveralls", + "hypothesis", "mock", "nltk", "numpy", "pandas", "pytest", + "pytest-cov", "scipy", "sklearn", "textblob", From 10f9a2f3567d4dea69af29a10241073987b4d6ab Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Tue, 28 Jul 2020 18:06:53 -0700 Subject: [PATCH 52/61] delete tests node for scrutinizer run --- .scrutinizer.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index d8a1c64c..eb1c0c9e 100755 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -25,7 +25,6 @@ build: - command: pylint-run use_website_config: true - tests: true checks: python: code_rating: true From 887b96386039fccd12dd943781728e18ad9b1c4a Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Tue, 11 Aug 2020 15:15:16 -0700 Subject: [PATCH 53/61] Update postman collection --- misc/TabPy.postman_collection.json | 102 +++++++++++++++++++---------- 1 file changed, 66 insertions(+), 36 deletions(-) diff --git a/misc/TabPy.postman_collection.json b/misc/TabPy.postman_collection.json index fb796356..9769f299 100755 --- a/misc/TabPy.postman_collection.json +++ b/misc/TabPy.postman_collection.json @@ -6,19 +6,16 @@ }, "item": [ { - "name": "{{endpoint}}/info", + "name": "{{host}}:{{port}}/info", "request": { "method": "GET", "header": [], - "body": { - "mode": "raw", - "raw": "" - }, "url": { - "raw": "{{endpoint}}/info", + "raw": "{{host}}:{{port}}/info", "host": [ - "{{endpoint}}" + "{{host}}" ], + "port": "{{port}}", "path": [ "info" ] @@ -27,8 +24,23 @@ "response": [] }, { - "name": "{{endpoint}}/evaluate", + "name": "{{host}}:{{port}}/evaluate", "request": { + "auth": { + "type": "basic", + "basic": [ + { + "key": "password", + "value": "P@ssw0rd", + "type": "string" + }, + { + "key": "username", + "value": "user1", + "type": "string" + } + ] + }, "method": "POST", "header": [ { @@ -36,19 +48,45 @@ "name": "Content-Type", "value": "application/json", "type": "text" + }, + { + "key": "TabPy-Client", + "value": "Postman for manual testing", + "type": "text" + }, + { + "key": "TabPy-User", + "value": "ogolovatyi", + "type": "text" } ], "body": { "mode": "raw", - "raw": "{\n\t\"data\": \n\t{ \n\t\t\"_arg1\" : [1, 2, 3], \n\t\t\"_arg2\" : [3, -1, 5]\n\t},\n\t\"script\": \n\t\"res = []\\nfor i in range(len(_arg1)):\\n res.append(_arg1[i] * _arg2[i])\\nreturn res\"\n}\n" + "raw": "{\n\t\"data\": \n\t{ \n\t\t\"_arg1\" : [1, 2, 3], \n\t\t\"_arg2\" : [3, -1, 5]\n\t},\n\t\"script\": \n\t\"return [x + y for x, y in zip(_arg1, _arg2)]\"\n}\n", + "options": { + "raw": {} + } }, "url": { - "raw": "{{endpoint}}/evaluate", + "raw": "{{host}}:{{port}}/evaluate", "host": [ - "{{endpoint}}" + "{{host}}" ], + "port": "{{port}}", "path": [ "evaluate" + ], + "query": [ + { + "key": "TabPy-Client", + "value": "Postman for Manual Testing", + "disabled": true + }, + { + "key": "TabPy-User", + "value": "ogolovatyi", + "disabled": true + } ] } }, @@ -59,15 +97,12 @@ "request": { "method": "GET", "header": [], - "body": { - "mode": "raw", - "raw": "" - }, "url": { - "raw": "{{endpoint}}/status", + "raw": "{{host}}:{{port}}/status", "host": [ - "{{endpoint}}" + "{{host}}" ], + "port": "{{port}}", "path": [ "status" ] @@ -76,19 +111,16 @@ "response": [] }, { - "name": "{{endpoint}}/endpoints", + "name": "{{host}}:{{port}}/endpoints", "request": { "method": "GET", "header": [], - "body": { - "mode": "raw", - "raw": "" - }, "url": { - "raw": "{{endpoint}}/endpoints", + "raw": "{{host}}:{{port}}/endpoints", "host": [ - "{{endpoint}}" + "{{host}}" ], + "port": "{{port}}", "path": [ "endpoints" ] @@ -97,7 +129,7 @@ "response": [] }, { - "name": "{{endpoint}}/query/add", + "name": "{{host}}:{{port}}/query/model_name", "request": { "method": "POST", "header": [ @@ -113,35 +145,33 @@ "raw": "{\r\n \"data\": {\r\n \"x\": [\r\n 6.35,\r\n 6.4,\r\n 6.65,\r\n 8.6,\r\n 8.9,\r\n 9,\r\n 9.1\r\n ],\r\n \"y\": [\r\n 1.95,\r\n 1.95,\r\n 2.05,\r\n 3.05,\r\n 3.05,\r\n 3.1,\r\n 3.15\r\n ]\r\n }\r\n}" }, "url": { - "raw": "{{endpoint}}/query/add", + "raw": "{{host}}:{{port}}/query/model_name", "host": [ - "{{endpoint}}" + "{{host}}" ], + "port": "{{port}}", "path": [ "query", - "add" + "model_name" ] } }, "response": [] }, { - "name": "{{endpoint}}/endpoints/add", + "name": "{{host}}:{{port}}/endpoints/model_name", "request": { "method": "GET", "header": [], - "body": { - "mode": "raw", - "raw": "" - }, "url": { - "raw": "{{endpoint}}/endpoints/add", + "raw": "{{host}}:{{port}}/endpoints/model_name", "host": [ - "{{endpoint}}" + "{{host}}" ], + "port": "{{port}}", "path": [ "endpoints", - "add" + "model_name" ] } }, From 6e46d315ebfa360ccb69c1bb57c80f9419fcdaa4 Mon Sep 17 00:00:00 2001 From: Oleksandr Golovatyi Date: Thu, 13 Aug 2020 16:22:25 -0700 Subject: [PATCH 54/61] remove print from error handling code (#439) * remove print from error handling code * remove & for linux cmd Co-authored-by: Olek Golovatyi --- tabpy/tabpy_server/common/default.conf | 2 +- tabpy/tabpy_server/handlers/base_handler.py | 4 ---- tests/integration/integ_test_base.py | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/tabpy/tabpy_server/common/default.conf b/tabpy/tabpy_server/common/default.conf index ee02453d..01aa274e 100644 --- a/tabpy/tabpy_server/common/default.conf +++ b/tabpy/tabpy_server/common/default.conf @@ -47,7 +47,7 @@ propagete=0 [handler_rootHandler] class=StreamHandler -level=DEBUG +level=INFO formatter=rootFormatter args=(sys.stdout,) diff --git a/tabpy/tabpy_server/handlers/base_handler.py b/tabpy/tabpy_server/handlers/base_handler.py index 5ba55546..5582909c 100644 --- a/tabpy/tabpy_server/handlers/base_handler.py +++ b/tabpy/tabpy_server/handlers/base_handler.py @@ -138,10 +138,6 @@ def error_out(self, code, log_message, info=None): self.set_status(code) self.write(json.dumps({"message": log_message, "info": info or {}})) - # We want to duplicate error message in console for - # loggers are misconfigured or causing the failure - # themselves - print(info) self.logger.log( logging.ERROR, 'Responding with status={}, message="{}", info="{}"'.format( diff --git a/tests/integration/integ_test_base.py b/tests/integration/integ_test_base.py index 9453a1f6..43ad9efe 100755 --- a/tests/integration/integ_test_base.py +++ b/tests/integration/integ_test_base.py @@ -169,7 +169,7 @@ def _get_config_file_name(self) -> str: config_file = open(os.path.join(self.tmp_dir, "test.conf"), "w+") config_file.write( "[TabPy]\n" - f"TABPY_QUERY_OBJECT_PATH = ./query_objects\n" + f"TABPY_QUERY_OBJECT_PATH = {self.tmp_dir}/query_objects\n" f"TABPY_PORT = {self._get_port()}\n" f"TABPY_STATE_PATH = {self.tmp_dir}\n" ) From 7eaccc10b01d87b7cb051a99469b0bf790fffe48 Mon Sep 17 00:00:00 2001 From: harold-xi <69485232+harold-xi@users.noreply.github.com> Date: Fri, 14 Aug 2020 14:20:05 -0700 Subject: [PATCH 55/61] Return HTTP 400 status when receiving a request with authentication credentials and authN is not configured (#440) * added 400 bad request response to event when authorization is not set up but user sends username and password * added unit tests to test what happens when there are credentials, but no authentication is required * renamed not_authorized flag to authentication_error flag * changed line formatting * changed the way auth error is handled --- tabpy/tabpy_server/handlers/base_handler.py | 78 +++++++++++------ .../tabpy_server/handlers/endpoint_handler.py | 13 +-- .../handlers/endpoints_handler.py | 9 +- .../handlers/evaluation_plane_handler.py | 5 +- .../handlers/query_plane_handler.py | 13 +-- .../handlers/service_info_handler.py | 6 +- tabpy/tabpy_server/handlers/status_handler.py | 6 +- .../handlers/upload_destination_handler.py | 5 +- tabpy/tabpy_server/handlers/util.py | 5 ++ .../server_tests/test_endpoint_handler.py | 51 +++++++++++ .../server_tests/test_endpoints_handler.py | 50 +++++++++++ .../test_evaluation_plane_handler.py | 84 +++++++++++++++++++ .../server_tests/test_service_info_handler.py | 18 +--- 13 files changed, 273 insertions(+), 70 deletions(-) diff --git a/tabpy/tabpy_server/handlers/base_handler.py b/tabpy/tabpy_server/handlers/base_handler.py index 5582909c..1296f4a1 100644 --- a/tabpy/tabpy_server/handlers/base_handler.py +++ b/tabpy/tabpy_server/handlers/base_handler.py @@ -6,6 +6,7 @@ import tornado.web from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters from tabpy.tabpy_server.handlers.util import hash_password +from tabpy.tabpy_server.handlers.util import AuthErrorStates import uuid @@ -132,7 +133,7 @@ def initialize(self, app): app.settings[SettingsParameters.LogRequestContext] ) self.logger.log(logging.DEBUG, "Checking if need to handle authentication") - self.not_authorized = not self.handle_authentication("v1") + self.auth_error = self.handle_authentication("v1") def error_out(self, code, log_message, info=None): self.set_status(code) @@ -360,7 +361,7 @@ def _validate_credentials(self, method) -> bool: ) return False - def handle_authentication(self, api_version) -> bool: + def handle_authentication(self, api_version): """ If authentication feature is configured checks provided credentials. @@ -372,27 +373,36 @@ def handle_authentication(self, api_version) -> bool: Returns ------- - bool - True if authentication is not required. - True if authentication is required and valid - credentials provided. - False otherwise. + String + None if authentication is not required and username and password are None. + None if authentication is required and valid credentials provided. + NotAuthorized if authenication is required and credentials are incorrect. + NotRequired if authentication is not required but credentials are provided. """ self.logger.log(logging.DEBUG, "Handling authentication") found, method = self._get_auth_method(api_version) if not found: - return False + return AuthErrorStates.NotAuthorized if method == "": - # Do not validate credentials - return True + if not self._get_basic_auth_credentials(): + self.logger.log(logging.DEBUG, + "authentication not required, username and password are none") + return AuthErrorStates.NONE + else: + self.logger.log(logging.DEBUG, + "authentication not required, username and password are not none") + return AuthErrorStates.NotRequired if not self._get_credentials(method): - return False + return AuthErrorStates.NotAuthorized - return self._validate_credentials(method) + if not self._validate_credentials(method): + return AuthErrorStates.NotAuthorized + + return AuthErrorStates.NONE - def should_fail_with_not_authorized(self): + def should_fail_with_auth_error(self): """ Checks if authentication is required: - if it is not returns false, None @@ -401,21 +411,35 @@ def should_fail_with_not_authorized(self): Returns ------- bool - False if authentication is not required or is - required and validation for credentials passes. - True if validation for credentials failed. + False if authentication is not required and username + and password is None or isrequired and validation + for credentials passes. + True if validation for credentials failed or + if authentication is not required and username and password + fields are not empty. """ - return self.not_authorized + return self.auth_error - def fail_with_not_authorized(self): + def fail_with_auth_error(self): """ - Prepares server 401 response. + Prepares server 401 response and server 400 response depending + on the value of the self.auth_error flag """ - self.logger.log(logging.ERROR, "Failing with 401 for unauthorized request") - self.set_status(401) - self.set_header("WWW-Authenticate", f'Basic realm="{self.tabpy_state.name}"') - self.error_out( - 401, - info="Unauthorized request.", - log_message="Invalid credentials provided.", - ) + if self.auth_error == AuthErrorStates.NotAuthorized: + self.logger.log(logging.ERROR, "Failing with 401 for unauthorized request") + self.set_status(401) + self.set_header("WWW-Authenticate", f'Basic realm="{self.tabpy_state.name}"') + self.error_out( + 401, + info="Unauthorized request.", + log_message="Invalid credentials provided.", + ) + else: + self.logger.log(logging.ERROR, "Failing with 400 for Bad Request") + self.set_status(400) + self.set_header("WWW-Authenticate", f'Basic realm="{self.tabpy_state.name}"') + self.error_out( + 400, + info="Bad request.", + log_message="Username or Password provided when authentication not available", + ) diff --git a/tabpy/tabpy_server/handlers/endpoint_handler.py b/tabpy/tabpy_server/handlers/endpoint_handler.py index 022d8e0b..583b1762 100644 --- a/tabpy/tabpy_server/handlers/endpoint_handler.py +++ b/tabpy/tabpy_server/handlers/endpoint_handler.py @@ -14,6 +14,7 @@ from tabpy.tabpy_server.handlers.base_handler import STAGING_THREAD from tabpy.tabpy_server.management.state import get_query_object_path from tabpy.tabpy_server.psws.callbacks import on_state_change +from tabpy.tabpy_server.handlers.util import AuthErrorStates from tornado import gen @@ -22,8 +23,8 @@ def initialize(self, app): super(EndpointHandler, self).initialize(app) def get(self, endpoint_name): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return self.logger.log(logging.DEBUG, f"Processing GET for /endpoints/{endpoint_name}") @@ -43,8 +44,8 @@ def get(self, endpoint_name): @gen.coroutine def put(self, name): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return self.logger.log(logging.DEBUG, f"Processing PUT for /endpoints/{name}") @@ -89,8 +90,8 @@ def put(self, name): @gen.coroutine def delete(self, name): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return self.logger.log(logging.DEBUG, f"Processing DELETE for /endpoints/{name}") diff --git a/tabpy/tabpy_server/handlers/endpoints_handler.py b/tabpy/tabpy_server/handlers/endpoints_handler.py index 66132dd2..bda8a16d 100644 --- a/tabpy/tabpy_server/handlers/endpoints_handler.py +++ b/tabpy/tabpy_server/handlers/endpoints_handler.py @@ -10,6 +10,7 @@ import logging from tabpy.tabpy_server.common.util import format_exception from tabpy.tabpy_server.handlers import ManagementHandler +from tabpy.tabpy_server.handlers.util import AuthErrorStates from tornado import gen @@ -18,8 +19,8 @@ def initialize(self, app): super(EndpointsHandler, self).initialize(app) def get(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return self._add_CORS_header() @@ -27,8 +28,8 @@ def get(self): @gen.coroutine def post(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return try: diff --git a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py index 13ee1175..2ad55568 100644 --- a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py @@ -6,6 +6,7 @@ import requests from tornado import gen from datetime import timedelta +from tabpy.tabpy_server.handlers.util import AuthErrorStates class RestrictedTabPy: @@ -100,8 +101,8 @@ def _post_impl(self): @gen.coroutine def post(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return self._add_CORS_header() diff --git a/tabpy/tabpy_server/handlers/query_plane_handler.py b/tabpy/tabpy_server/handlers/query_plane_handler.py index aab42593..c66e9fa9 100644 --- a/tabpy/tabpy_server/handlers/query_plane_handler.py +++ b/tabpy/tabpy_server/handlers/query_plane_handler.py @@ -13,6 +13,7 @@ from tabpy.tabpy_server.common.util import format_exception import urllib from tornado import gen +from tabpy.tabpy_server.handlers.util import AuthErrorStates def _get_uuid(): @@ -67,8 +68,8 @@ def _query(self, po_name, data, uid, qry): # don't check API key (client does not send or receive data for OPTIONS, # it just allows the client to subsequently make a POST request) def options(self, pred_name): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return self.logger.log(logging.DEBUG, f"Processing OPTIONS for /query/{pred_name}") @@ -212,8 +213,8 @@ def _get_actual_model(self, endpoint_name): @gen.coroutine def get(self, endpoint_name): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return start = time.time() @@ -224,8 +225,8 @@ def get(self, endpoint_name): def post(self, endpoint_name): self.logger.log(logging.DEBUG, f"Processing POST for /query/{endpoint_name}...") - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return start = time.time() diff --git a/tabpy/tabpy_server/handlers/service_info_handler.py b/tabpy/tabpy_server/handlers/service_info_handler.py index 51152a98..5ba1469d 100644 --- a/tabpy/tabpy_server/handlers/service_info_handler.py +++ b/tabpy/tabpy_server/handlers/service_info_handler.py @@ -1,15 +1,15 @@ import json from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters from tabpy.tabpy_server.handlers import ManagementHandler - +from tabpy.tabpy_server.handlers.util import AuthErrorStates class ServiceInfoHandler(ManagementHandler): def initialize(self, app): super(ServiceInfoHandler, self).initialize(app) def get(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return self._add_CORS_header() diff --git a/tabpy/tabpy_server/handlers/status_handler.py b/tabpy/tabpy_server/handlers/status_handler.py index 2f743b3f..493ca036 100644 --- a/tabpy/tabpy_server/handlers/status_handler.py +++ b/tabpy/tabpy_server/handlers/status_handler.py @@ -1,15 +1,15 @@ import json import logging from tabpy.tabpy_server.handlers import BaseHandler - +from tabpy.tabpy_server.handlers.util import AuthErrorStates class StatusHandler(BaseHandler): def initialize(self, app): super(StatusHandler, self).initialize(app) def get(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return self._add_CORS_header() diff --git a/tabpy/tabpy_server/handlers/upload_destination_handler.py b/tabpy/tabpy_server/handlers/upload_destination_handler.py index 729aff3e..d6757dd1 100644 --- a/tabpy/tabpy_server/handlers/upload_destination_handler.py +++ b/tabpy/tabpy_server/handlers/upload_destination_handler.py @@ -1,6 +1,7 @@ from tabpy.tabpy_server.app.SettingsParameters import SettingsParameters from tabpy.tabpy_server.handlers import ManagementHandler import os +from tabpy.tabpy_server.handlers.util import AuthErrorStates _QUERY_OBJECT_STAGING_FOLDER = "staging" @@ -11,8 +12,8 @@ def initialize(self, app): super(UploadDestinationHandler, self).initialize(app) def get(self): - if self.should_fail_with_not_authorized(): - self.fail_with_not_authorized() + if self.should_fail_with_auth_error() != AuthErrorStates.NONE: + self.fail_with_auth_error() return path = self.settings[SettingsParameters.StateFilePath] diff --git a/tabpy/tabpy_server/handlers/util.py b/tabpy/tabpy_server/handlers/util.py index 14e029c6..a5ed8719 100644 --- a/tabpy/tabpy_server/handlers/util.py +++ b/tabpy/tabpy_server/handlers/util.py @@ -1,6 +1,11 @@ import binascii from hashlib import pbkdf2_hmac +from enum import Enum, auto +class AuthErrorStates(Enum): + NONE = auto() + NotAuthorized = auto() + NotRequired = auto() def hash_password(username, pwd): """ diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index a9393c42..8834e05f 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -111,3 +111,54 @@ def test_valid_creds_unknown_endpoint_fails(self): }, ) self.assertEqual(404, response.code) + +class TestEndpointHandlerWithoutAuth(AsyncHTTPTestCase): + @classmethod + def setUpClass(cls): + _init_asyncio_patch() + prefix = "__TestEndpointHandlerWithoutAuth_" + + # create state.ini dir and file + cls.state_dir = tempfile.mkdtemp(prefix=prefix) + cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") + cls.state_file.write( + "[Service Info]\n" + "Name = TabPy Serve\n" + "Description = \n" + "Creation Time = 0\n" + "Access-Control-Allow-Origin = \n" + "Access-Control-Allow-Headers = \n" + "Access-Control-Allow-Methods = \n" + "\n" + "[Query Objects Service Versions]\n" + "\n" + "[Query Objects Docstrings]\n" + "\n" + "[Meta]\n" + "Revision Number = 1\n" + ) + cls.state_file.close() + + @classmethod + def tearDownClass(cls): + os.remove(cls.state_file.name) + os.rmdir(cls.state_dir) + + def get_app(self): + self.app = TabPyApp(None) + return self.app._create_tornado_web_app() + + def test_creds_no_auth_fails(self): + response = self.fetch( + "/endpoints/", + method="GET", + headers={ + "Authorization": "Basic {}".format( + base64.b64encode("username:password".encode("utf-8")).decode( + "utf-8" + ) + ) + }, + ) + self.assertEqual(400, response.code) + \ No newline at end of file diff --git a/tests/unit/server_tests/test_endpoints_handler.py b/tests/unit/server_tests/test_endpoints_handler.py index 6ae00fc7..5255b2b4 100755 --- a/tests/unit/server_tests/test_endpoints_handler.py +++ b/tests/unit/server_tests/test_endpoints_handler.py @@ -94,3 +94,53 @@ def test_valid_creds_pass(self): }, ) self.assertEqual(200, response.code) + + +class TestEndpointsHandlerWithoutAuth(AsyncHTTPTestCase): + @classmethod + def setUpClass(cls): + prefix = "__TestEndpointsHandlerWithoutAuth_" + + # create state.ini dir and file + cls.state_dir = tempfile.mkdtemp(prefix=prefix) + cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") + cls.state_file.write( + "[Service Info]\n" + "Name = TabPy Serve\n" + "Description = \n" + "Creation Time = 0\n" + "Access-Control-Allow-Origin = \n" + "Access-Control-Allow-Headers = \n" + "Access-Control-Allow-Methods = \n" + "\n" + "[Query Objects Service Versions]\n" + "\n" + "[Query Objects Docstrings]\n" + "\n" + "[Meta]\n" + "Revision Number = 1\n" + ) + cls.state_file.close() + + @classmethod + def tearDownClass(cls): + os.remove(cls.state_file.name) + os.rmdir(cls.state_dir) + + def get_app(self): + self.app = TabPyApp(None) + return self.app._create_tornado_web_app() + + def test_creds_no_auth_fails(self): + response = self.fetch( + "/endpoints", + method="GET", + headers={ + "Authorization": "Basic {}".format( + base64.b64encode("username:password".encode("utf-8")).decode( + "utf-8" + ) + ) + }, + ) + self.assertEqual(400, response.code) diff --git a/tests/unit/server_tests/test_evaluation_plane_handler.py b/tests/unit/server_tests/test_evaluation_plane_handler.py index 49b67dfb..346a3e60 100755 --- a/tests/unit/server_tests/test_evaluation_plane_handler.py +++ b/tests/unit/server_tests/test_evaluation_plane_handler.py @@ -203,3 +203,87 @@ def test_script_returns_none(self): }) self.assertEqual(200, response.code) self.assertEqual(b'null', response.body) + + +class TestEvaluationPlainHandlerWithoutAuth(AsyncHTTPTestCase): + @classmethod + def setUpClass(cls): + prefix = "__TestEvaluationPlainHandlerWithoutAuth_" + + # create state.ini dir and file + cls.state_dir = tempfile.mkdtemp(prefix=prefix) + cls.state_file = open(os.path.join(cls.state_dir, "state.ini"), "w+") + cls.state_file.write( + "[Service Info]\n" + "Name = TabPy Serve\n" + "Description = \n" + "Creation Time = 0\n" + "Access-Control-Allow-Origin = \n" + "Access-Control-Allow-Headers = \n" + "Access-Control-Allow-Methods = \n" + "\n" + "[Query Objects Service Versions]\n" + "\n" + "[Query Objects Docstrings]\n" + "\n" + "[Meta]\n" + "Revision Number = 1\n" + ) + cls.state_file.close() + + cls.script = ( + '{"data":{"_arg1":[2,3],"_arg2":[3,-1]},' + '"script":"res=[]\\nfor i in range(len(_arg1)):\\n ' + 'res.append(_arg1[i] * _arg2[i])\\nreturn res"}' + ) + + cls.script_not_present = ( + '{"data":{"_arg1":[2,3],"_arg2":[3,-1]},' + '"":"res=[]\\nfor i in range(len(_arg1)):\\n ' + 'res.append(_arg1[i] * _arg2[i])\\nreturn res"}' + ) + + cls.args_not_present = ( + '{"script":"res=[]\\nfor i in range(len(_arg1)):\\n ' + 'res.append(_arg1[i] * _arg2[i])\\nreturn res"}' + ) + + cls.args_not_sequential = ( + '{"data":{"_arg1":[2,3],"_arg3":[3,-1]},' + '"script":"res=[]\\nfor i in range(len(_arg1)):\\n ' + 'res.append(_arg1[i] * _arg3[i])\\nreturn res"}' + ) + + cls.nan_coverts_to_null =\ + '{"data":{"_arg1":[2,3],"_arg2":[3,-1]},'\ + '"script":"return [float(1), float(\\"NaN\\"), float(2)]"}' + + cls.script_returns_none = ( + '{"data":{"_arg1":[2,3],"_arg2":[3,-1]},' + '"script":"return None"}' + ) + + @classmethod + def tearDownClass(cls): + os.remove(cls.state_file.name) + os.rmdir(cls.state_dir) + + def get_app(self): + self.app = TabPyApp(None) + return self.app._create_tornado_web_app() + + def test_creds_no_auth_fails(self): + response = self.fetch( + "/evaluate", + method="POST", + body=self.script, + headers={ + "Authorization": "Basic {}".format( + base64.b64encode("username:password".encode("utf-8")).decode( + "utf-8" + ) + ) + }, + ) + self.assertEqual(400, response.code) + \ No newline at end of file diff --git a/tests/unit/server_tests/test_service_info_handler.py b/tests/unit/server_tests/test_service_info_handler.py index 767a50c0..8751ee8a 100644 --- a/tests/unit/server_tests/test_service_info_handler.py +++ b/tests/unit/server_tests/test_service_info_handler.py @@ -137,20 +137,4 @@ def test_given_server_with_no_auth_and_password_expect_correct_info_response(sel } response = self.fetch("/info", headers=header) - self.assertEqual(response.code, 200) - actual_response = json.loads(response.body) - expected_response = _create_expected_info_response( - self.app.settings, self.app.tabpy_state - ) - - self.assertDictEqual(actual_response, expected_response) - self.assertTrue("versions" in actual_response) - versions = actual_response["versions"] - self.assertTrue("v1" in versions) - v1 = versions["v1"] - self.assertTrue("features" in v1) - features = v1["features"] - self.assertDictEqual( - {}, - features, - ) + self.assertEqual(response.code, 400) From 885b7394f444f3ea543c13f1cdde2b0c435a3973 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Fri, 14 Aug 2020 14:25:58 -0700 Subject: [PATCH 56/61] Version to 2.0.0 --- CHANGELOG | 12 ++++++++++++ tabpy/VERSION | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG b/CHANGELOG index ad197662..40ce5d5c 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,5 +1,17 @@ # Changelog +## v2.0.0 + +### Breaking changes + +- TabPy fails with 400 when it is not configure for authentication + but credentials are provided by client. + +### Bug fixes + +- When TabPy is running with no console attached it is not failing + with 500 when trying to respond with 401 status. + ## v1.2.0 ### Improvements diff --git a/tabpy/VERSION b/tabpy/VERSION index 867e5243..359a5b95 100755 --- a/tabpy/VERSION +++ b/tabpy/VERSION @@ -1 +1 @@ -1.2.0 \ No newline at end of file +2.0.0 \ No newline at end of file From cf0d2ffae1cc98c09e9c2183ef6080b66d7ccc3e Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Fri, 14 Aug 2020 14:27:40 -0700 Subject: [PATCH 57/61] Version to 2.0.0 --- CHANGELOG | 2 -- 1 file changed, 2 deletions(-) diff --git a/CHANGELOG b/CHANGELOG index 40ce5d5c..e3987150 100755 --- a/CHANGELOG +++ b/CHANGELOG @@ -12,8 +12,6 @@ - When TabPy is running with no console attached it is not failing with 500 when trying to respond with 401 status. -## v1.2.0 - ### Improvements - Minor code cleanup. From 4bc8e8a7631087e6f215f6b8a6736c30d85d3f86 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Fri, 14 Aug 2020 14:36:18 -0700 Subject: [PATCH 58/61] Fix codystyle warnings --- tabpy/tabpy_server/handlers/base_handler.py | 12 ++++++------ tabpy/tabpy_server/handlers/util.py | 2 ++ tests/unit/server_tests/test_endpoint_handler.py | 3 ++- .../server_tests/test_evaluation_plane_handler.py | 2 +- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tabpy/tabpy_server/handlers/base_handler.py b/tabpy/tabpy_server/handlers/base_handler.py index 1296f4a1..8e330067 100644 --- a/tabpy/tabpy_server/handlers/base_handler.py +++ b/tabpy/tabpy_server/handlers/base_handler.py @@ -373,7 +373,7 @@ def handle_authentication(self, api_version): Returns ------- - String + String None if authentication is not required and username and password are None. None if authentication is required and valid credentials provided. NotAuthorized if authenication is required and credentials are incorrect. @@ -387,11 +387,11 @@ def handle_authentication(self, api_version): if method == "": if not self._get_basic_auth_credentials(): self.logger.log(logging.DEBUG, - "authentication not required, username and password are none") + "authentication not required, username and password are none") return AuthErrorStates.NONE else: self.logger.log(logging.DEBUG, - "authentication not required, username and password are not none") + "authentication not required, username and password are not none") return AuthErrorStates.NotRequired if not self._get_credentials(method): @@ -399,7 +399,7 @@ def handle_authentication(self, api_version): if not self._validate_credentials(method): return AuthErrorStates.NotAuthorized - + return AuthErrorStates.NONE def should_fail_with_auth_error(self): @@ -412,9 +412,9 @@ def should_fail_with_auth_error(self): ------- bool False if authentication is not required and username - and password is None or isrequired and validation + and password is None or isrequired and validation for credentials passes. - True if validation for credentials failed or + True if validation for credentials failed or if authentication is not required and username and password fields are not empty. """ diff --git a/tabpy/tabpy_server/handlers/util.py b/tabpy/tabpy_server/handlers/util.py index a5ed8719..ae9d7387 100644 --- a/tabpy/tabpy_server/handlers/util.py +++ b/tabpy/tabpy_server/handlers/util.py @@ -2,11 +2,13 @@ from hashlib import pbkdf2_hmac from enum import Enum, auto + class AuthErrorStates(Enum): NONE = auto() NotAuthorized = auto() NotRequired = auto() + def hash_password(username, pwd): """ Hashes password using PKDBF2 method: diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index 8834e05f..f74cd3c7 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -112,6 +112,7 @@ def test_valid_creds_unknown_endpoint_fails(self): ) self.assertEqual(404, response.code) + class TestEndpointHandlerWithoutAuth(AsyncHTTPTestCase): @classmethod def setUpClass(cls): @@ -161,4 +162,4 @@ def test_creds_no_auth_fails(self): }, ) self.assertEqual(400, response.code) - \ No newline at end of file + \ No newline at end of file diff --git a/tests/unit/server_tests/test_evaluation_plane_handler.py b/tests/unit/server_tests/test_evaluation_plane_handler.py index 346a3e60..983f1ff8 100755 --- a/tests/unit/server_tests/test_evaluation_plane_handler.py +++ b/tests/unit/server_tests/test_evaluation_plane_handler.py @@ -286,4 +286,4 @@ def test_creds_no_auth_fails(self): }, ) self.assertEqual(400, response.code) - \ No newline at end of file + \ No newline at end of file From 1028df4b3346a36421c910020a6cb74e9cd567d4 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Fri, 14 Aug 2020 15:09:59 -0700 Subject: [PATCH 59/61] Restore scrutinizer settings --- .scrutinizer.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.scrutinizer.yml b/.scrutinizer.yml index eb1c0c9e..d8a1c64c 100755 --- a/.scrutinizer.yml +++ b/.scrutinizer.yml @@ -25,6 +25,7 @@ build: - command: pylint-run use_website_config: true + tests: true checks: python: code_rating: true From 366f6e09943cfe5d6eabcaa4295496e0437906d6 Mon Sep 17 00:00:00 2001 From: Olek Golovatyi Date: Fri, 14 Aug 2020 15:12:10 -0700 Subject: [PATCH 60/61] Fix codestyle --- tests/unit/server_tests/test_endpoint_handler.py | 2 +- tests/unit/server_tests/test_evaluation_plane_handler.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index f74cd3c7..31b778ed 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -162,4 +162,4 @@ def test_creds_no_auth_fails(self): }, ) self.assertEqual(400, response.code) - \ No newline at end of file + \ No newline at end of file diff --git a/tests/unit/server_tests/test_evaluation_plane_handler.py b/tests/unit/server_tests/test_evaluation_plane_handler.py index 983f1ff8..52b0d98a 100755 --- a/tests/unit/server_tests/test_evaluation_plane_handler.py +++ b/tests/unit/server_tests/test_evaluation_plane_handler.py @@ -286,4 +286,3 @@ def test_creds_no_auth_fails(self): }, ) self.assertEqual(400, response.code) - \ No newline at end of file From 753257feb2e13c467d54d19a85b9fa65f844c9c6 Mon Sep 17 00:00:00 2001 From: harold-xi <69485232+harold-xi@users.noreply.github.com> Date: Wed, 26 Aug 2020 14:40:22 -0700 Subject: [PATCH 61/61] Error loading state: AccessDenied: Access Denied (AWS S3 backend) #18801 (#443) * added 400 bad request response to event when authorization is not set up but user sends username and password * added unit tests to test what happens when there are credentials, but no authentication is required * renamed not_authorized flag to authentication_error flag * changed line formatting * changed the way auth error is handled * changed changelog and verison number and made styling updates * tabpy.query now passes all headers in the request * got rid of print statement * changed code styling * removed duplicate class * code cleanup --- .../handlers/evaluation_plane_handler.py | 8 +-- tabpy/tabpy_server/handlers/util.py | 2 - .../deploy_and_evaluate_model_auth.conf | 57 +++++++++++++++++++ .../test_deploy_and_evaluate_model_auth_on.py | 34 +++++++++++ .../server_tests/test_endpoint_handler.py | 1 - 5 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 tests/integration/resources/deploy_and_evaluate_model_auth.conf create mode 100644 tests/integration/test_deploy_and_evaluate_model_auth_on.py diff --git a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py index 2ad55568..5170f925 100644 --- a/tabpy/tabpy_server/handlers/evaluation_plane_handler.py +++ b/tabpy/tabpy_server/handlers/evaluation_plane_handler.py @@ -10,18 +10,19 @@ class RestrictedTabPy: - def __init__(self, protocol, port, logger, timeout): + def __init__(self, protocol, port, logger, timeout, headers): self.protocol = protocol self.port = port self.logger = logger self.timeout = timeout + self.headers = headers def query(self, name, *args, **kwargs): url = f"{self.protocol}://localhost:{self.port}/query/{name}" self.logger.log(logging.DEBUG, f"Querying {url}...") internal_data = {"data": args or kwargs} data = json.dumps(internal_data) - headers = {"content-type": "application/json"} + headers = self.headers response = requests.post( url=url, data=data, headers=headers, timeout=self.timeout, verify=False ) @@ -73,7 +74,6 @@ def _post_impl(self): "the format _arg1, _arg2, _argN", ) return - function_to_evaluate = f"def _user_script(tabpy{arguments_str}):\n" for u in user_code.splitlines(): function_to_evaluate += " " + u + "\n" @@ -126,7 +126,7 @@ def post(self): @gen.coroutine def _call_subprocess(self, function_to_evaluate, arguments): restricted_tabpy = RestrictedTabPy( - self.protocol, self.port, self.logger, self.eval_timeout + self.protocol, self.port, self.logger, self.eval_timeout, self.request.headers ) # Exec does not run the function, so it does not block. exec(function_to_evaluate, globals()) diff --git a/tabpy/tabpy_server/handlers/util.py b/tabpy/tabpy_server/handlers/util.py index ae9d7387..a5ed8719 100644 --- a/tabpy/tabpy_server/handlers/util.py +++ b/tabpy/tabpy_server/handlers/util.py @@ -2,13 +2,11 @@ from hashlib import pbkdf2_hmac from enum import Enum, auto - class AuthErrorStates(Enum): NONE = auto() NotAuthorized = auto() NotRequired = auto() - def hash_password(username, pwd): """ Hashes password using PKDBF2 method: diff --git a/tests/integration/resources/deploy_and_evaluate_model_auth.conf b/tests/integration/resources/deploy_and_evaluate_model_auth.conf new file mode 100644 index 00000000..a15293fc --- /dev/null +++ b/tests/integration/resources/deploy_and_evaluate_model_auth.conf @@ -0,0 +1,57 @@ +[TabPy] +# TABPY_QUERY_OBJECT_PATH = /tmp/query_objects +TABPY_PORT = 9009 +# TABPY_STATE_PATH = ./tabpy/tabpy_server + +# Where static pages live +# TABPY_STATIC_PATH = ./tabpy/tabpy_server/static + +# For how to configure TabPy authentication read +# Authentication section in docs/server-config.md. +TABPY_PWD_FILE = ./tests/integration/resources/pwdfile.txt + +# To set up secure TabPy uncomment and modify the following lines. +# Note only PEM-encoded x509 certificates are supported. +# TABPY_TRANSFER_PROTOCOL = https +# TABPY_CERTIFICATE_FILE = path/to/certificate/file.crt +# TABPY_KEY_FILE = path/to/key/file.key + +# Log additional request details including caller IP, full URL, client +# end user info if provided. +# TABPY_LOG_DETAILS = true + +# Configure how long a custom script provided to the /evaluate method +# will run before throwing a TimeoutError. +# The value should be a float representing the timeout time in seconds. +#TABPY_EVALUATE_TIMEOUT = 30 + +[loggers] +keys=root + +[handlers] +keys=rootHandler,rotatingFileHandler + +[formatters] +keys=rootFormatter + +[logger_root] +level=DEBUG +handlers=rootHandler,rotatingFileHandler +qualname=root +propagete=0 + +[handler_rootHandler] +class=StreamHandler +level=DEBUG +formatter=rootFormatter +args=(sys.stdout,) + +[handler_rotatingFileHandler] +class=handlers.RotatingFileHandler +level=DEBUG +formatter=rootFormatter +args=('tabpy_log.log', 'a', 1000000, 5) + +[formatter_rootFormatter] +format=%(asctime)s [%(levelname)s] (%(filename)s:%(module)s:%(lineno)d): %(message)s +datefmt=%Y-%m-%d,%H:%M:%S diff --git a/tests/integration/test_deploy_and_evaluate_model_auth_on.py b/tests/integration/test_deploy_and_evaluate_model_auth_on.py new file mode 100644 index 00000000..56f92793 --- /dev/null +++ b/tests/integration/test_deploy_and_evaluate_model_auth_on.py @@ -0,0 +1,34 @@ +from . import integ_test_base + + +class TestDeployAndEvaluateModelAuthOn(integ_test_base.IntegTestBase): + def _get_config_file_name(self) -> str: + return "./tests/integration/resources/deploy_and_evaluate_model_auth.conf" + + def _get_port(self) -> str: + return "9009" + + def test_deploy_and_evaluate_model(self): + # Uncomment the following line to preserve + # test case output and other files (config, state, ect.) + # in system temp folder. + # self.set_delete_temp_folder(False) + + self.deploy_models(self._get_username(), self._get_password()) + + headers = { + "Content-Type": "application/json", + "Authorization": "Basic dXNlcjE6UEBzc3cwcmQ=", + "Host": "localhost:9009", + } + payload = """{ + "data": { "_arg1": ["happy", "sad", "neutral"] }, + "script": + "return tabpy.query('Sentiment Analysis',_arg1)['response']" + }""" + + conn = self._get_connection() + conn.request("POST", "/evaluate", payload, headers) + SentimentAnalysis_eval = conn.getresponse() + self.assertEqual(200, SentimentAnalysis_eval.status) + SentimentAnalysis_eval.read() diff --git a/tests/unit/server_tests/test_endpoint_handler.py b/tests/unit/server_tests/test_endpoint_handler.py index 31b778ed..2f2d20c8 100755 --- a/tests/unit/server_tests/test_endpoint_handler.py +++ b/tests/unit/server_tests/test_endpoint_handler.py @@ -162,4 +162,3 @@ def test_creds_no_auth_fails(self): }, ) self.assertEqual(400, response.code) - \ No newline at end of file