From bcd9b4812528af59b6ff219136b4ea421e1e1997 Mon Sep 17 00:00:00 2001 From: RamanjaneyuluIdavalapati Date: Fri, 22 Jun 2018 19:58:45 +0200 Subject: [PATCH 1/4] kwikapi client supporting cluster as header --- .travis.yml | 4 ++-- kwikapi/api.py | 1 + kwikapi/client.py | 7 +++++-- setup.py | 2 +- 4 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1b9b562..a37d1ee 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,8 @@ deploy: - LICENSE - kwikapi/api.py - kwikapi/__init__.py - name: kwikapi-0.2.8 - tag_name: 0.2.8 + name: kwikapi-0.2.9 + tag_name: 0.2.9 on: repo: deep-compute/kwikapi - provider: pypi diff --git a/kwikapi/api.py b/kwikapi/api.py index c12c9e5..d512262 100644 --- a/kwikapi/api.py +++ b/kwikapi/api.py @@ -27,6 +27,7 @@ PROTOCOL_HEADER = 'X-KwikAPI-Protocol' REQUEST_ID_HEADER = 'X-KwikAPI-RequestID' TIMING_HEADER = 'X-KwikAPI-Timing' +CLUSTER_HEADER = 'X-KwikAPI-Cluster' class Counter: def __init__(self, v=0): diff --git a/kwikapi/client.py b/kwikapi/client.py index e473e42..fed8a40 100644 --- a/kwikapi/client.py +++ b/kwikapi/client.py @@ -6,7 +6,7 @@ from .protocols import PROTOCOLS from .exception import APICallFailed -from .api import PROTOCOL_HEADER, REQUEST_ID_HEADER +from .api import PROTOCOL_HEADER, REQUEST_ID_HEADER, CLUSTER_HEADER from .utils import get_loggable_params DUMMY_LOG = Dummy() @@ -43,11 +43,12 @@ def map_url(self, url): class Client: DEFAULT_PROTOCOL = 'pickle' - def __init__(self, url, version=None, protocol=DEFAULT_PROTOCOL, + def __init__(self, url, cluster=None, version=None, protocol=DEFAULT_PROTOCOL, path=None, request='', timeout=None, dnscache=None, log=DUMMY_LOG): self._url = url + self.cluster = cluster self._version = version self._protocol = protocol # FIXME: check validity @@ -76,6 +77,8 @@ def _prepare_request(self, post_body, get_params=None): headers[PROTOCOL_HEADER] = self._protocol if self._request: headers[REQUEST_ID_HEADER] = self._request.id + if self.cluster: + headers[CLUSTER_HEADER] = repr(self.cluster) upath = [self._version] + self._path upath = '/'.join(x for x in upath if x) diff --git a/setup.py b/setup.py index 9001df9..1f2a86b 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '0.2.8' +version = '0.2.9' setup( name="kwikapi", version=version, From 7c569b667973c111fd5d108a9828b170ec81e6b8 Mon Sep 17 00:00:00 2001 From: RamanjaneyuluIdavalapati Date: Tue, 3 Jul 2018 12:01:08 +0200 Subject: [PATCH 2/4] returning Content Length in headers --- .travis.yml | 4 ++-- kwikapi/api.py | 1 - kwikapi/client.py | 24 +++++++++++++++--------- kwikapi/protocols.py | 4 +++- setup.py | 6 +++--- 5 files changed, 23 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index a37d1ee..58ce598 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,8 @@ deploy: - LICENSE - kwikapi/api.py - kwikapi/__init__.py - name: kwikapi-0.2.9 - tag_name: 0.2.9 + name: kwikapi-0.3 + tag_name: 0.3 on: repo: deep-compute/kwikapi - provider: pypi diff --git a/kwikapi/api.py b/kwikapi/api.py index d512262..c12c9e5 100644 --- a/kwikapi/api.py +++ b/kwikapi/api.py @@ -27,7 +27,6 @@ PROTOCOL_HEADER = 'X-KwikAPI-Protocol' REQUEST_ID_HEADER = 'X-KwikAPI-RequestID' TIMING_HEADER = 'X-KwikAPI-Timing' -CLUSTER_HEADER = 'X-KwikAPI-Cluster' class Counter: def __init__(self, v=0): diff --git a/kwikapi/client.py b/kwikapi/client.py index fed8a40..7e82f71 100644 --- a/kwikapi/client.py +++ b/kwikapi/client.py @@ -6,7 +6,7 @@ from .protocols import PROTOCOLS from .exception import APICallFailed -from .api import PROTOCOL_HEADER, REQUEST_ID_HEADER, CLUSTER_HEADER +from .api import PROTOCOL_HEADER, REQUEST_ID_HEADER from .utils import get_loggable_params DUMMY_LOG = Dummy() @@ -43,12 +43,11 @@ def map_url(self, url): class Client: DEFAULT_PROTOCOL = 'pickle' - def __init__(self, url, cluster=None, version=None, protocol=DEFAULT_PROTOCOL, + def __init__(self, url, version=None, protocol=DEFAULT_PROTOCOL, path=None, request='', timeout=None, dnscache=None, - log=DUMMY_LOG): + headers=None, log=DUMMY_LOG): self._url = url - self.cluster = cluster self._version = version self._protocol = protocol # FIXME: check validity @@ -56,6 +55,7 @@ def __init__(self, url, cluster=None, version=None, protocol=DEFAULT_PROTOCOL, self._request = request self._timeout = timeout self._dnscache = dnscache + self._headers = headers self._log = log if not self._dnscache: @@ -65,7 +65,8 @@ def _get_state(self): return dict(url=self._url, version=self._version, protocol=self._protocol, path=self._path, request=self._request, timeout=self._timeout, - dnscache=self._dnscache, log=self._log) + dnscache=self._dnscache, headers=self._headers, + log=self._log) def _copy(self, **kwargs): _kwargs = self._get_state() @@ -73,12 +74,17 @@ def _copy(self, **kwargs): return Client(**_kwargs) def _prepare_request(self, post_body, get_params=None): - headers = {} - headers[PROTOCOL_HEADER] = self._protocol + headers = (self._headers or {}).copy() + if self._request: + for hk, hv in self._request.headers.items(): + if not hk.lower().startswith('x-kwikapi-'): + continue + headers[hk] = hv + headers[REQUEST_ID_HEADER] = self._request.id - if self.cluster: - headers[CLUSTER_HEADER] = repr(self.cluster) + + headers[PROTOCOL_HEADER] = self._protocol upath = [self._version] + self._path upath = '/'.join(x for x in upath if x) diff --git a/kwikapi/protocols.py b/kwikapi/protocols.py index 0815cf2..15dd586 100644 --- a/kwikapi/protocols.py +++ b/kwikapi/protocols.py @@ -44,7 +44,9 @@ def get_name(): @staticmethod def serialize(data): data = to_python_type(data) - return json.dumps(data) + data = json.dumps(data) + + return data.encode('utf-8') @staticmethod def deserialize(data): diff --git a/setup.py b/setup.py index 1f2a86b..7c497cd 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,6 @@ from setuptools import setup, find_packages -version = '0.2.9' +version = '0.3' setup( name="kwikapi", version=version, @@ -22,8 +22,8 @@ ], extras_require={ 'django': ['kwikapi-django==0.2.2'], - 'tornado': ['kwikapi-tornado==0.2.4'], - 'all': ['kwikapi-django==0.2.2', 'kwikapi-tornado==0.2.4'] + 'tornado': ['kwikapi-tornado==0.2.5'], + 'all': ['kwikapi-django==0.2.2', 'kwikapi-tornado==0.2.5'] }, classifiers=[ 'Environment :: Web Environment', From 04810cf281a780fe3445766cb2423e7e8f54699c Mon Sep 17 00:00:00 2001 From: RamanjaneyuluIdavalapati Date: Tue, 3 Jul 2018 12:29:58 +0200 Subject: [PATCH 3/4] testcases solved --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 57fef7f..e9d41bd 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ as living documentation and test cases, we use `MockRequest` so we don't need >>> api.register(Calc(), "v1") # `v1` is the version of this example >>> req = MockRequest(url="/api/v1/add?a=10&b=20") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 30 @@ -120,14 +120,14 @@ We can register the same class with different versions (for testing here) >>> api.register(Calc(), "v2") >>> req = MockRequest(url="/api/v1/add?a=10&b=20") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 30 >>> res['success'] True >>> req = MockRequest(url="/api/v2/add?a=10&b=20") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 30 >>> res['success'] @@ -155,14 +155,14 @@ We can register different classes with different versions >>> api.register(ConcStr(), "v2") >>> req = MockRequest(url="/api/v1/add?a=10&b=20") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 30 >>> res['success'] True >>> req = MockRequest(url="/api/v2/add?a=in&b=dia") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 'india' >>> res['success'] @@ -186,7 +186,7 @@ We can specify the default version so that when you don't mention version in the >>> api.register(Calc(), "v1") >>> req = MockRequest(url="/api/add?a=10&b=20") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 30 @@ -300,14 +300,14 @@ Register methods with different namespaces >>> api.register(ConcStr(), "v1", "Calc/ConcStr") >>> req = MockRequest(url="/api/v1/Calc/add?a=10&b=20") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 30 >>> res['success'] True >>> req = MockRequest(url="/api/v1/Calc/ConcStr/add?a=in&b=dia") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 'india' >>> res['success'] @@ -334,7 +334,7 @@ Register same methods with same version with different namespaces >>> api.register(CalcScintific(), "v1", "scintific") >>> req = MockRequest(url="/api/v1/add?a=10&b=20") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 30 @@ -342,7 +342,7 @@ Register same methods with same version with different namespaces True >>> req = MockRequest(url="/api/v1/scintific/add?a=10&b=20") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 40 @@ -371,7 +371,7 @@ User can change the response if he wants it >>> api.register(Calc(), "v1") >>> req = MockRequest(url="/api/v1/add?a=10&b=20") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> res['result'] 30 @@ -463,7 +463,7 @@ KwikAPI supports JSON, Messagepack, Pickle and Numpy protocols >>> base.register_protocol(CustomProtocol()) >>> req = MockRequest(url="/api/v1/add?a=10&b=20") ->>> res = json.loads(base.handle_request(req)) +>>> res = json.loads(base.handle_request(req).decode('utf-8')) >>> res['result'] 30 @@ -530,7 +530,7 @@ To check API methods under specific version and namespace we can provide URL as >>> api.register(Calc(), "v1", "calc") >>> req = MockRequest(url="/api/v1/apidoc") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> pprint(res['result']) {'namespace': {"('v1', 'calc')": {'add': {'doc': None, @@ -564,7 +564,7 @@ To check API methods under specific version and namespace we can provide URL as True >>> req = MockRequest(url="/api/v1/apidoc?version=v1&namespace=calc") ->>> res = json.loads(BaseRequestHandler(api).handle_request(req)) +>>> res = json.loads(BaseRequestHandler(api).handle_request(req).decode('utf-8')) >>> pprint(res['result']) {'add': {'doc': None, 'gives_stream': False, From 8e802d7f107296afef871162cdfea60b8fc97539 Mon Sep 17 00:00:00 2001 From: RamanjaneyuluIdavalapati Date: Tue, 3 Jul 2018 21:17:08 +0200 Subject: [PATCH 4/4] CaseInsensitiveDict for request and response headers --- kwikapi/api.py | 6 ++++-- kwikapi/client.py | 5 +++-- setup.py | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/kwikapi/api.py b/kwikapi/api.py index c12c9e5..c9c0036 100644 --- a/kwikapi/api.py +++ b/kwikapi/api.py @@ -12,6 +12,7 @@ import concurrent.futures from deeputil import Dummy, AttrDict, generate_random_string +from requests.structures import CaseInsensitiveDict from .protocols import PROTOCOLS, DEFAULT_PROTOCOL from .apidoc import ApiDoc @@ -126,7 +127,8 @@ class MockRequest(BaseRequest): def __init__(self, **kwargs): super().__init__() - self._request = dict(method='GET', body='', headers={}) + self._request = dict(method='GET', body='', + headers=CaseInsensitiveDict()) self._request.update(kwargs) self.response = MockResponse() @@ -149,7 +151,7 @@ def headers(self): class MockResponse(BaseResponse): def __init__(self): super().__init__() - self.headers = {} + self.headers = CaseInsensitiveDict() self.raw_response = None def write(self, data, protocol, stream=False): diff --git a/kwikapi/client.py b/kwikapi/client.py index 7e82f71..74a552f 100644 --- a/kwikapi/client.py +++ b/kwikapi/client.py @@ -1,6 +1,7 @@ import socket from urllib.parse import urljoin, urlparse, urlencode import urllib.request +from requests.structures import CaseInsensitiveDict from deeputil import Dummy, ExpiringCache @@ -55,7 +56,7 @@ def __init__(self, url, version=None, protocol=DEFAULT_PROTOCOL, self._request = request self._timeout = timeout self._dnscache = dnscache - self._headers = headers + self._headers = CaseInsensitiveDict(headers) if headers is not None else None self._log = log if not self._dnscache: @@ -74,7 +75,7 @@ def _copy(self, **kwargs): return Client(**_kwargs) def _prepare_request(self, post_body, get_params=None): - headers = (self._headers or {}).copy() + headers = (self._headers or CaseInsensitiveDict()).copy() if self._request: for hk, hv in self._request.headers.items(): diff --git a/setup.py b/setup.py index 7c497cd..de98fe5 100644 --- a/setup.py +++ b/setup.py @@ -22,8 +22,8 @@ ], extras_require={ 'django': ['kwikapi-django==0.2.2'], - 'tornado': ['kwikapi-tornado==0.2.5'], - 'all': ['kwikapi-django==0.2.2', 'kwikapi-tornado==0.2.5'] + 'tornado': ['kwikapi-tornado==0.2.6'], + 'all': ['kwikapi-django==0.2.2', 'kwikapi-tornado==0.2.6'] }, classifiers=[ 'Environment :: Web Environment',