Merge pull request #13 from plq/master

Add caching support

CHANGELOG

@@ -0,0 +1,14 @@
+0.3
+===
+* Add package caching
+
+
+0.2
+===
+* Add support for Spyne 2.9 APIs.
+
+
+0.1
+===
+
+* First working version

setup.py

@@ -30,7 +30,7 @@
     license='GPL',
 
     install_requires=[
-        "spyne<2.99", "sqlalchemy<0.8", "werkzeug", "twisted",
+        "spyne>=2.10", "sqlalchemy<0.8", "werkzeug", "twisted",
     ],
     include_package_data=True,
     entry_points = {

spynepi/__init__.py

@@ -19,5 +19,5 @@
 # MA 02110-1301, USA.
 #
 
-__version__ = '0.1'
+__version__ = '0.3'
 

spynepi/entity/html.py

@@ -19,15 +19,22 @@
 # MA 02110-1301, USA.
 #
 
+import logging
+logger = logging.getLogger(__name__)
+
 import os
-import subprocess
+import shutil
+import tempfile
 
 from lxml import html
 from sqlalchemy import sql
 
+from spyne.error import ValidationError
+from spyne.util import reconstruct_url
+
 from pkg_resources import resource_filename
 
-from setuptools.command.easy_install import main as easy_install
+from sqlalchemy.orm.exc import NoResultFound
 
 from spyne.decorator import rpc
 from spyne.error import RequestNotAllowed
@@ -54,49 +61,110 @@
 class IndexService(ServiceBase):
     @rpc (_returns=Array(Index), _patterns=[HttpPattern("/",verb="GET")])
     def index(ctx):
-        idx = []
-        packages = ctx.udc.session.query(Package).all()
-        for package in packages:
-            idx.append(Index(
+        return [Index(
                 Updated=package.package_cdate,
                 Package=AnyUri.Value(text=package.package_name,
                     href=package.releases[-1].rdf_about),
                 Description=package.package_description,
-            ))
+            ) for package in ctx.udc.session.query(Package)]
+
+
+def cache_package(spec, own_url):
+    from glob import glob
+    from setuptools.command.easy_install import main as easy_install
+    from distutils.core import run_setup
+    import ConfigParser
+
+    path = tempfile.mkdtemp('.spynepi')
+    easy_install(["--user", "-U", "--editable", "--build-directory",
+                                                           path, spec])
+
+    if os.environ.has_key('HOME'):
+        rc = os.path.join(os.environ['HOME'], '.pypirc')
+        config = ConfigParser.ConfigParser()
+
+        if os.path.exists(rc):
+            config.read(rc)
+
+        try:
+            config.add_section(REPO_NAME)
+
+            config.set(REPO_NAME, 'repository', own_url)
+            config.set(REPO_NAME, 'username', 'x')
+            config.set(REPO_NAME, 'password', 'y')
+
+        except ConfigParser.DuplicateSectionError:
+            pass
+
+        try:
+            config.add_section('distutils')
+        except ConfigParser.DuplicateSectionError:
+            pass
+
+        try:
+            index_servers = config.get('distutils', 'index-servers')
+            index_servers = index_servers.split('\n')
+            if 'spynepi' not in index_servers:
+                index_servers.append(REPO_NAME)
+
+        except ConfigParser.NoOptionError:
+            index_servers = [REPO_NAME]
+
+        config.set('distutils', 'index-servers', '\n'.join(index_servers))
+
+        config.write(open(rc,'w'))
+
+    else: # FIXME: ??? No idea. Hopefully setuptools knows better.
+        pass # raise NotImplementedError("$HOME not defined, .pypirc not found.")
 
-        return idx
+    try:
+        # plagiarized from setuptools
+        setups = glob(os.path.join(path, '*', 'setup.py'))
+        if not setups:
+            raise ValidationError(
+                "Couldn't find a setup script in %r editable distribution: %r" %
+                                                    (spec, os.path.join(path,'*'))
+            )
 
+        if len(setups)>1:
+            raise ValidationError(
+                "Multiple setup scripts in found in %r editable distribution: %r" %
+                                                    (spec, setups)
+            )
 
-def cache_packages(project_name):
-    path = os.path.join(FILES_PATH,"files","tmp")
-    if not os.path.exists(path):
-        os.makedirs(path)
+        lib_dir =os.path.dirname(setups[0])
+        os.chdir(lib_dir)
+        dist = run_setup(setups[0])
+        dist.commands = ['register', 'sdist', 'upload']
+        dist.command_options = {
+            'register': {'repository': ('command line', 'spynepi')},
+            'upload': {'repository': ('command line', 'spynepi')},
+            'sdist': {},
+        }
 
-    easy_install(["--user","-U","--build-directory",path,project_name])
-    dpath = os.path.join(path,project_name)
-    dpath = os.path.abspath(dpath)
+        dist.run_commands()
 
-    if not dpath.startswith(path):
-        # This request tried to read arbitrary data from the filesystem
-        raise RequestNotAllowed(repr([project_name,]))
 
-    command = ["python", "setup.py", "register", "-r", REPO_NAME, "sdist",
-                                                "upload", "-r", REPO_NAME]
-    subprocess.call(command, cwd=dpath)
+    finally:
+        shutil.rmtree(path)
 
 
 class HtmlService(ServiceBase):
-    @rpc(Unicode, Unicode,_returns=Unicode, _patterns=[
+    @rpc(Unicode, Unicode, _returns=Unicode, _patterns=[
             HttpPattern("/<project_name>"),
             HttpPattern("/<project_name>/"),
             HttpPattern("/<project_name>/<version>"),
             HttpPattern("/<project_name>/<version>/"),
         ])
-    def download_html(ctx,project_name,version):
+    def download_html(ctx, project_name, version):
         ctx.transport.mime_type = "text/html"
 
-        ctx.udc.session.query(Package).filter_by(
+        try:
+            ctx.udc.session.query(Package).filter_by(
                                                 package_name=project_name).one()
+        except NoResultFound:
+            cache_package(project_name, reconstruct_url(ctx.transport.req_env,
+                                                  path=False, query_string=False))
 
         download = HtmlPage(TPL_DOWNLOAD)
         download.title = project_name

spynepi/entity/root.py

@@ -29,6 +29,7 @@
 
 from spyne.decorator import rpc
 from spyne.error import ArgumentError
+from spyne.error import ResourceNotFoundError
 from spyne.model.primitive import String
 from spyne.model.primitive import Unicode
 from spyne.service import ServiceBase
@@ -133,6 +134,8 @@ def package_content():
                         Release.release_version == version)
                     ).first()
 
+                if rel is None:
+                    raise ResourceNotFoundError(name)
                 rel.distributions.append(generate_dist())
                 package_content()
 

spynepi/main.py

@@ -72,6 +72,7 @@ def call_wrapper(self, ctx):
             return Application.call_wrapper(self, ctx)
 
         except NoResultFound, e:
+            logger.exception(e)
             ctx.out_string = ["Resource not found"]
             raise ResourceNotFoundError() # Return HTTP 404
 
@@ -82,14 +83,14 @@ def main(connection_string=DB_CONNECTION_STRING):
     logging.getLogger('spyne.protocol.xml').setLevel(logging.DEBUG)
     #logging.getLogger('sqlalchemy.engine.base.Engine').setLevel(logging.DEBUG)
 
-    index_app = MyApplication([RootService, IndexService],"http://usefulinc.com/ns/doap#",
-                                in_protocol=HttpRpc(), out_protocol=HtmlTable())
+    index_app = MyApplication([RootService, IndexService], "http://usefulinc.com/ns/doap#",
+                            in_protocol=HttpRpc(), out_protocol=HtmlTable())
 
-    rdf_app = MyApplication([RdfService],"http://usefulinc.com/ns/doap#",
-                                in_protocol=HttpRpc(), out_protocol=XmlDocument())
+    rdf_app = MyApplication([RdfService], "http://usefulinc.com/ns/doap#",
+                            in_protocol=HttpRpc(), out_protocol=XmlDocument())
 
     html_app = MyApplication([HtmlService],"http://usefulinc.com/ns/doap#",
-                                in_protocol=HttpRpc(), out_protocol=HttpRpc())
+                            in_protocol=HttpRpc(), out_protocol=HttpRpc())
 
     db_handle = init_database(connection_string)
 
@@ -107,6 +108,7 @@ def _on_method_call(ctx):
     # this is called once all data is sent to the client.
     def _on_method_return_object(ctx):
         ctx.udc.session.commit()
+
     def _on_wsgi_close(ctx):
         if ctx.udc is not None:
             ctx.udc.close()
@@ -122,7 +124,6 @@ def _on_wsgi_close(ctx):
     for a in wsgi_index,wsgi_rdf,wsgi_html:
         a.event_manager.add_listener('wsgi_close', _on_wsgi_close)
 
-
     url_map = Map([Rule("/", endpoint=wsgi_index),
         Rule("/<project_name>", endpoint=wsgi_html),
         Rule("/<project_name>/", endpoint=wsgi_html),