Merge pull request #47 from RamanjaneyuluIdavalapati/master

Asyncio for threading

.travis.yml

@@ -16,8 +16,8 @@ deploy:
     - LICENSE
     - kwikapi/api.py
     - kwikapi/__init__.py
-  name: kwikapi-0.2.4
-  tag_name: 0.2.4
+  name: kwikapi-0.2.5
+  tag_name: 0.2.5
   on:
     repo: deep-compute/kwikapi
 - provider: pypi

README.md

@@ -507,6 +507,75 @@ $ wget "http://localhost:8888/api/v1/add" --header="X-KwikAPI-Protocol: pickle"
 $ wget "http://localhost:8888/api/v1/add" --header="X-KwikAPI-Protocol: numpy" --post-file /tmp/data.numpy
 ```
 
+### API Doc
+Using API Doc we can look at what are the all API methods available
+
+To see available API methods the URL will be http://localhost:8888/api/v1/apidoc for default version
+
+To check API methods under specific version we can provide URL as http://localhost:8888/api/v1/apidoc?version=<-version->
+
+To check API methods under specific version and namespace we can provide URL as http://localhost:8888/api/v1/apidoc?version=<-version->&namespace=<-namespace->
+
+```python
+>>> import json
+>>> from pprint import pprint
+
+>>> from kwikapi import API, MockRequest, BaseRequestHandler
+
+>>> class Calc(object):
+...    def add(self, a: int, b: int) -> int:
+...        return a + b
+
+>>> api = API()
+>>> api.register(Calc(), "v1", "calc")
+
+>>> req = MockRequest(url="/api/v1/apidoc")
+>>> res = json.loads(BaseRequestHandler(api).handle_request(req))
+
+>>> pprint(res['result'])
+{'namespace': {"('v1', 'calc')": {'add': {'doc': None,
+                                          'gives_stream': False,
+                                          'params': {'a': {'default': None,
+                                                           'required': True,
+                                                           'type': 'int'},
+                                                     'b': {'default': None,
+                                                           'required': True,
+                                                           'type': 'int'}},
+                                          'return_type': 'int'}}},
+ 'version': {'v1': {'add': {'doc': None,
+                            'gives_stream': False,
+                            'params': {'a': {'default': None,
+                                             'required': True,
+                                             'type': 'int'},
+                                       'b': {'default': None,
+                                             'required': True,
+                                             'type': 'int'}},
+                            'return_type': 'int'},
+                    'apidoc': {'doc': None,
+                               'gives_stream': False,
+                               'params': {'namespace': {'default': None,
+                                                        'required': False,
+                                                        'type': 'str'},
+                                          'version': {'default': None,
+                                                      'required': False,
+                                                      'type': 'str'}},
+                               'return_type': 'dict'}}}}
+>>> res['success']
+True
+
+>>> req = MockRequest(url="/api/v1/apidoc?version=v1&namespace=calc")
+>>> res = json.loads(BaseRequestHandler(api).handle_request(req))
+>>> pprint(res['result'])
+{'add': {'doc': None,
+         'gives_stream': False,
+         'params': {'a': {'default': None, 'required': True, 'type': 'int'},
+                    'b': {'default': None, 'required': True, 'type': 'int'}},
+         'return_type': 'int'}}
+>>> res['success']
+True
+
+```
+
 ### Bulk request handling
 It will be very convenient if the user has facility to make bulk requests.
 When making a large number of requests, the overhead of network latency and HTTP request/response processing

kwikapi/api.py

@@ -25,7 +25,8 @@
 DUMMY_LOG = Dummy()
 
 PROTOCOL_HEADER = 'X-KwikAPI-Protocol'
-NETPATH_HEADER = 'X-KwikAPI-Netpath'
+REQUEST_ID_HEADER = 'X-KwikAPI-RequestID'
+TIMING_HEADER = 'X-KwikAPI-Timing'
 
 class Counter:
     def __init__(self, v=0):
@@ -50,7 +51,14 @@ def __init__(self):
         self.fn_params = None
         self.response = None
         self.protocol = None
-        self.id = generate_random_string(length=5).decode('utf8')
+        self._id = generate_random_string(length=5).decode('utf8')
+
+    @property
+    def id(self):
+        _id = self.headers.get(REQUEST_ID_HEADER, '')
+        if _id:
+            return '{}.{}'.format(_id, self._id)
+        return _id
 
     @abc.abstractproperty
     def url(self):
@@ -174,13 +182,13 @@ class API(object):
 
     THREADPOOL_SIZE = 32
 
-    def __init__(self, default_version=None, id='',
+    def __init__(self, default_version=None, _id='',
             threadpool=None, threadpool_size=THREADPOOL_SIZE,
             log=DUMMY_LOG):
 
         self._api_funcs = {}
-        self.log = log.bind(api_id=id)
-        self.id = id
+        self.log = log.bind(api_id=_id)
+        self._id = _id
         self.default_version = default_version
 
         self.threadpool = None
@@ -189,8 +197,7 @@ def __init__(self, default_version=None, id='',
             self.threadpool = threadpool
         else:
             if threadpool_size:
-                pass
-                #self.threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=threadpool_size)
+                self.threadpool = concurrent.futures.ThreadPoolExecutor(max_workers=threadpool_size)
 
         self.register(ApiDoc(self._api_funcs), "v1")
 
@@ -378,15 +385,10 @@ def _resolve_call_info(self, request):
         namespace = namespace if namespace else None
         r.namespace = namespace or ''
 
-        request.netpath = '{netpath}=>{id}_{reqid}({namespace}/{function})'.format(
-            netpath=request.headers.get(NETPATH_HEADER, ''),
-            id=self.api.id,
-            reqid=request.id,
-            namespace=r.namespace,
-            function=fn_name,
-        )
-
-        request.log = self.log.bind(__netpath=request.netpath)
+        request.log = self.log.bind(__requestid=request._id,
+                namespace=r.namespace,
+                function=fn_name,
+                apiid=self.api._id)
 
         query_string = urlp.query
 
@@ -445,7 +447,6 @@ def _find_request_protocol(self, request):
         return self.PROTOCOLS[protocol]
 
     def handle_request(self, request):
-        #import pdb; pdb.set_trace()
         protocol = self._find_request_protocol(request)
         request.protocol = protocol.get_name()
         response = request.response
@@ -459,6 +460,8 @@ def handle_request(self, request):
             result = request.fn(**request.fn_params)
             tcompute = time.time() - tcompute
 
+            response.headers[TIMING_HEADER] = str(tcompute)
+
             # Serialize the response
             if request.fn.__func__.func_info['gives_stream']:
                 n, t = response.write(result, protocol, stream=True)
@@ -474,8 +477,9 @@ def handle_request(self, request):
 
         except Exception as e:
             message = e.message if hasattr(e, 'message') else str(e)
-            message = '[(%s) %s: %s]' % (self.api.id, e.__class__.__name__, message)
+            message = '[(%s) %s: %s]' % (self.api._id, e.__class__.__name__, message)
 
+            _log = request.log if hasattr(request, 'log') else self.log
             self.log.exception('handle_request_error', message=message)
             response.write(dict(success=False, message=message), protocol)
 

kwikapi/apidoc.py

@@ -1,9 +1,21 @@
+import typing
+
 from .exception import UnknownVersionOrNamespace, UnknownVersion
 
 class ApiDoc(object):
+
     def __init__(self, api_funcs):
         self.api_funcs = api_funcs
 
+    def _type_str(self, t):
+        if t is None:
+            return str(None)
+
+        if isinstance(t, typing.TypingMeta):
+            return str(t)
+
+        return t.__name__
+
     def apidoc(self, version: str=None, namespace: str=None) -> dict:
         versions = {}
         namespaces = {}
@@ -15,9 +27,18 @@ def apidoc(self, version: str=None, namespace: str=None) -> dict:
             # So we are converting types to strings
             # FIXME: Need to handle in a better way
 
+            # FIXME: remove this after 'req' in info is removed/resolved
+            fninfo = fninfo.copy()
+            fninfo['info'] = fninfo['info'].copy()
+            fninfo['info']['params'] = fninfo['info']['params'].copy()
+            if 'req' in fninfo['info']:
+                fninfo['info'].pop('req')
+
             for key in fninfo['info']['params'].keys():
-                fninfo['info']['params'][key]['type'] = str(fninfo['info']['params'][key]['type'])
-            fninfo['info']['return_type'] = str(fninfo['info']['return_type'])
+                _type = fninfo['info']['params'][key]['type']
+                fninfo['info']['params'][key] = fninfo['info']['params'][key].copy()
+                fninfo['info']['params'][key]['type'] = self._type_str(_type)
+            fninfo['info']['return_type'] = self._type_str(fninfo['info']['return_type'])
 
             vfns = versions.get(ver, {})
             vfns[fn_name] = fninfo['info']

kwikapi/client.py

@@ -6,7 +6,7 @@
 
 from .protocols import PROTOCOLS
 from .exception import APICallFailed
-from .api import PROTOCOL_HEADER, NETPATH_HEADER
+from .api import PROTOCOL_HEADER, REQUEST_ID_HEADER
 from .utils import get_loggable_params
 
 DUMMY_LOG = Dummy()
@@ -36,23 +36,23 @@ def map_url(self, url):
             path = '{}?{}'.format(path, query)
 
         if port:
-            return '{}://{}:{}{}'.format(scheme, host, port, path)
+            return '{}://{}:{}{}'.format(scheme, _host, port, path)
         else:
-            return '{}://{}{}'.format(scheme, host, path)
+            return '{}://{}{}'.format(scheme, _host, path)
 
 class Client:
     DEFAULT_PROTOCOL = 'pickle'
 
     def __init__(self, url, version=None, protocol=DEFAULT_PROTOCOL,
-            path=None, netpath='', timeout=None, dnscache=None,
+            path=None, request='', timeout=None, dnscache=None,
             log=DUMMY_LOG):
 
         self._url = url
         self._version = version
         self._protocol = protocol # FIXME: check validity
 
         self._path = path or []
-        self._netpath = netpath
+        self._request = request
         self._timeout = timeout
         self._dnscache = dnscache
         self._log = log
@@ -63,7 +63,7 @@ def __init__(self, url, version=None, protocol=DEFAULT_PROTOCOL,
     def _get_state(self):
         return dict(url=self._url, version=self._version,
             protocol=self._protocol, path=self._path,
-            netpath=self._netpath, timeout=self._timeout,
+            request=self._request, timeout=self._timeout,
             dnscache=self._dnscache, log=self._log)
 
     def _copy(self, **kwargs):
@@ -74,7 +74,8 @@ def _copy(self, **kwargs):
     def _prepare_request(self, post_body, get_params=None):
         headers = {}
         headers[PROTOCOL_HEADER] = self._protocol
-        headers[NETPATH_HEADER] = self._netpath
+        if self._request:
+            headers[REQUEST_ID_HEADER] = self.request.id
 
         upath = [self._version] + self._path
         upath = '/'.join(x for x in upath if x)

setup.py

@@ -1,6 +1,6 @@
 from setuptools import setup, find_packages
 
-version = '0.2.4'
+version = '0.2.5'
 setup(
     name="kwikapi",
     version=version,
@@ -22,8 +22,8 @@
     ],
     extras_require={
         'django': ['kwikapi-django==0.2.2'],
-        'tornado': ['kwikapi-tornado==0.2.2'],
-        'all': ['kwikapi-django==0.2.2', 'kwikapi-tornado==0.2.2']
+        'tornado': ['kwikapi-tornado==0.2.3'],
+        'all': ['kwikapi-django==0.2.2', 'kwikapi-tornado==0.2.3']
     },
     classifiers=[
         'Environment :: Web Environment',