From 128d5700134a14a0b2374651c018ad17a09d578b Mon Sep 17 00:00:00 2001 From: Burak Arslan Date: Sat, 17 Nov 2012 15:21:53 +0200 Subject: [PATCH 01/10] steal yolk's pypi module --- spynepi/entity/html.py | 2 +- spynepi/util/__init__.py | 0 spynepi/util/pypi.py | 326 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 spynepi/util/__init__.py create mode 100644 spynepi/util/pypi.py diff --git a/spynepi/entity/html.py b/spynepi/entity/html.py index 7c893af..b40310f 100644 --- a/spynepi/entity/html.py +++ b/spynepi/entity/html.py @@ -72,7 +72,7 @@ def cache_packages(project_name): if not os.path.exists(path): os.makedirs(path) - easy_install(["--user","-U","--build-directory",path,project_name]) + easy_install(["--user", "-U", "--build-directory", path, project_name]) dpath = os.path.join(path,project_name) dpath = os.path.abspath(dpath) diff --git a/spynepi/util/__init__.py b/spynepi/util/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/spynepi/util/pypi.py b/spynepi/util/pypi.py new file mode 100644 index 0000000..3ba3ba0 --- /dev/null +++ b/spynepi/util/pypi.py @@ -0,0 +1,326 @@ + +""" + +pypi.py +======= + +Desc: Library for getting information about Python packages by querying + The CheeseShop (PYPI a.k.a. Python Package Index). + + +Author: Rob Cakebread + +License : BSD (See COPYING) + +""" + +__docformat__ = 'restructuredtext' + +import re +import platform +if platform.python_version().startswith('2'): + import xmlrpclib + import cPickle + import urllib2 +else: + import xmlrpc.client as xmlrpclib + import pickle + import urllib.request as urllib2 +import os +import time +import logging +import urllib + +from yolk.utils import get_yolk_dir + + +XML_RPC_SERVER = 'http://pypi.python.org/pypi' + +class addinfourl(urllib2.addinfourl): + """ + Replacement addinfourl class compatible with python-2.7's xmlrpclib + + In python-2.7, xmlrpclib expects that the response object that it receives + has a getheader method. httplib.HTTPResponse provides this but + urllib2.addinfourl does not. Add the necessary functions here, ported to + use the internal data structures of addinfourl. + """ + + def getheader(self, name, default=None): + if self.headers is None: + raise httplib.ResponseNotReady() + return self.headers.getheader(name, default) + + def getheaders(self): + if self.headers is None: + raise httplib.ResponseNotReady() + return self.headers.items() + +urllib2.addinfourl = addinfourl + + +class ProxyTransport(xmlrpclib.Transport): + """ + Provides an XMl-RPC transport routing via a http proxy. + + This is done by using urllib2, which in turn uses the environment + varable http_proxy and whatever else it is built to use (e.g. the + windows registry). + + NOTE: the environment variable http_proxy should be set correctly. + See check_proxy_setting() below. + + Written from scratch but inspired by xmlrpc_urllib_transport.py + file from http://starship.python.net/crew/jjkunce/ by jjk. + + A. Ellerton 2006-07-06 + """ + + def request(self, host, handler, request_body, verbose): + '''Send xml-rpc request using proxy''' + #We get a traceback if we don't have this attribute: + self.verbose = verbose + url = 'http://' + host + handler + request = urllib2.Request(url) + request.add_data(request_body) + # Note: 'Host' and 'Content-Length' are added automatically + request.add_header('User-Agent', self.user_agent) + request.add_header('Content-Type', 'text/xml') + proxy_handler = urllib2.ProxyHandler() + opener = urllib2.build_opener(proxy_handler) + fhandle = opener.open(request) + return(self.parse_response(fhandle)) + + +def check_proxy_setting(): + """ + If the environmental variable 'HTTP_PROXY' is set, it will most likely be + in one of these forms: + + proxyhost:8080 + http://proxyhost:8080 + + urlllib2 requires the proxy URL to start with 'http://' + This routine does that, and returns the transport for xmlrpc. + """ + try: + http_proxy = os.environ['HTTP_PROXY'] + except KeyError: + return + + if not http_proxy.startswith('http://'): + match = re.match('(http://)?([-_\.A-Za-z]+):(\d+)', http_proxy) + #if not match: + # raise Exception('Proxy format not recognised: [%s]' % http_proxy) + os.environ['HTTP_PROXY'] = 'http://%s:%s' % (match.group(2), + match.group(3)) + return + + +class CheeseShop(object): + + """Interface to Python Package Index""" + + def __init__(self, debug=False, no_cache=False, yolk_dir=None): + self.debug = debug + self.no_cache = no_cache + if yolk_dir: + self.yolk_dir = yolk_dir + else: + self.yolk_dir = get_yolk_dir() + self.xmlrpc = self.get_xmlrpc_server() + self.pkg_cache_file = self.get_pkg_cache_file() + self.last_sync_file = self.get_last_sync_file() + self.pkg_list = None + self.logger = logging.getLogger("yolk") + self.get_cache() + + def get_cache(self): + """ + Get a package name list from disk cache or PyPI + """ + #This is used by external programs that import `CheeseShop` and don't + #want a cache file written to ~/.pypi and query PyPI every time. + if self.no_cache: + self.pkg_list = self.list_packages() + return + + if not os.path.exists(self.yolk_dir): + os.mkdir(self.yolk_dir) + if os.path.exists(self.pkg_cache_file): + self.pkg_list = self.query_cached_package_list() + else: + self.logger.debug("DEBUG: Fetching package list cache from PyPi...") + self.fetch_pkg_list() + + def get_last_sync_file(self): + """ + Get the last time in seconds since The Epoc since the last pkg list sync + """ + return os.path.abspath(self.yolk_dir + "/last_sync") + + def get_xmlrpc_server(self): + """ + Returns PyPI's XML-RPC server instance + """ + check_proxy_setting() + if os.environ.has_key('XMLRPC_DEBUG'): + debug = 1 + else: + debug = 0 + try: + return xmlrpclib.Server(XML_RPC_SERVER, transport=ProxyTransport(), verbose=debug) + except IOError: + self.logger("ERROR: Can't connect to XML-RPC server: %s" \ + % XML_RPC_SERVER) + + def get_pkg_cache_file(self): + """ + Returns filename of pkg cache + """ + return os.path.abspath('%s/pkg_list.pkl' % self.yolk_dir) + + def query_versions_pypi(self, package_name): + """Fetch list of available versions for a package from The CheeseShop""" + if not package_name in self.pkg_list: + self.logger.debug("Package %s not in cache, querying PyPI..." \ + % package_name) + self.fetch_pkg_list() + #I have to set version=[] for edge cases like "Magic file extensions" + #but I'm not sure why this happens. It's included with Python or + #because it has a space in it's name? + versions = [] + for pypi_pkg in self.pkg_list: + if pypi_pkg.lower() == package_name.lower(): + if self.debug: + self.logger.debug("DEBUG: %s" % package_name) + versions = self.package_releases(pypi_pkg) + package_name = pypi_pkg + break + return (package_name, versions) + + def query_cached_package_list(self): + """Return list of pickled package names from PYPI""" + if self.debug: + self.logger.debug("DEBUG: reading pickled cache file") + return cPickle.load(open(self.pkg_cache_file, "r")) + + def fetch_pkg_list(self): + """Fetch and cache master list of package names from PYPI""" + self.logger.debug("DEBUG: Fetching package name list from PyPI") + package_list = self.list_packages() + cPickle.dump(package_list, open(self.pkg_cache_file, "w")) + self.pkg_list = package_list + + def search(self, spec, operator): + '''Query PYPI via XMLRPC interface using search spec''' + return self.xmlrpc.search(spec, operator.lower()) + + def changelog(self, hours): + '''Query PYPI via XMLRPC interface using search spec''' + return self.xmlrpc.changelog(get_seconds(hours)) + + def updated_releases(self, hours): + '''Query PYPI via XMLRPC interface using search spec''' + return self.xmlrpc.updated_releases(get_seconds(hours)) + + def list_packages(self): + """Query PYPI via XMLRPC interface for a a list of all package names""" + return self.xmlrpc.list_packages() + + def release_urls(self, package_name, version): + """Query PYPI via XMLRPC interface for a pkg's available versions""" + + return self.xmlrpc.release_urls(package_name, version) + + def release_data(self, package_name, version): + """Query PYPI via XMLRPC interface for a pkg's metadata""" + try: + return self.xmlrpc.release_data(package_name, version) + except xmlrpclib.Fault: + #XXX Raises xmlrpclib.Fault if you give non-existant version + #Could this be server bug? + return + + def package_releases(self, package_name): + """Query PYPI via XMLRPC interface for a pkg's available versions""" + if self.debug: + self.logger.debug("DEBUG: querying PyPI for versions of " \ + + package_name) + return self.xmlrpc.package_releases(package_name) + + def get_download_urls(self, package_name, version="", pkg_type="all"): + """Query PyPI for pkg download URI for a packge""" + + if version: + versions = [version] + else: + + #If they don't specify version, show em all. + + (package_name, versions) = self.query_versions_pypi(package_name) + + all_urls = [] + for ver in versions: + metadata = self.release_data(package_name, ver) + for urls in self.release_urls(package_name, ver): + if pkg_type == "source" and urls['packagetype'] == "sdist": + all_urls.append(urls['url']) + elif pkg_type == "egg" and \ + urls['packagetype'].startswith("bdist"): + all_urls.append(urls['url']) + elif pkg_type == "all": + #All + all_urls.append(urls['url']) + + #Try the package's metadata directly in case there's nothing + #returned by XML-RPC's release_urls() + if metadata and metadata.has_key('download_url') and \ + metadata['download_url'] != "UNKNOWN" and \ + metadata['download_url'] != None: + if metadata['download_url'] not in all_urls: + if pkg_type != "all": + url = filter_url(pkg_type, metadata['download_url']) + if url: + all_urls.append(url) + return all_urls + +def filter_url(pkg_type, url): + """ + Returns URL of specified file type + 'source', 'egg', or 'all' + """ + bad_stuff = ["?modtime", "#md5="] + for junk in bad_stuff: + if junk in url: + url = url.split(junk)[0] + break + + #pkg_spec==dev (svn) + if url.endswith("-dev"): + url = url.split("#egg=")[0] + + if pkg_type == "all": + return url + + elif pkg_type == "source": + valid_source_types = [".tgz", ".tar.gz", ".zip", ".tbz2", ".tar.bz2"] + for extension in valid_source_types: + if url.lower().endswith(extension): + return url + + elif pkg_type == "egg": + if url.lower().endswith(".egg"): + return url + +def get_seconds(hours): + """ + Get number of seconds since epoch from now minus `hours` + + @param hours: Number of `hours` back in time we are checking + @type hours: int + + Return integer for number of seconds for now minus hours + + """ + return int(time.time() - (60 * 60) * hours) From 4585f3e077b437e59d84dab8990046c276a13f58 Mon Sep 17 00:00:00 2001 From: Burak Arslan Date: Sat, 17 Nov 2012 17:02:53 +0200 Subject: [PATCH 02/10] refactor. --- spynepi/entity/html.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/spynepi/entity/html.py b/spynepi/entity/html.py index b40310f..008b533 100644 --- a/spynepi/entity/html.py +++ b/spynepi/entity/html.py @@ -54,17 +54,13 @@ class IndexService(ServiceBase): @rpc (_returns=Array(Index), _patterns=[HttpPattern("/",verb="GET")]) def index(ctx): - idx = [] - packages = ctx.udc.session.query(Package).all() - for package in packages: - idx.append(Index( + return [Index( Updated=package.package_cdate, Package=AnyUri.Value(text=package.package_name, href=package.releases[-1].rdf_about), Description=package.package_description, - )) + ) for package in ctx.udc.session.query(Package)] - return idx def cache_packages(project_name): From dfe21a11cc163bee27edf74f98e2c043dd3c3bfa Mon Sep 17 00:00:00 2001 From: Burak Arslan Date: Sat, 17 Nov 2012 17:03:08 +0200 Subject: [PATCH 03/10] prune the unneeded from the pypi module. --- spynepi/util/pypi.py | 91 ++++++++++---------------------------------- 1 file changed, 21 insertions(+), 70 deletions(-) diff --git a/spynepi/util/pypi.py b/spynepi/util/pypi.py index 3ba3ba0..f4bd453 100644 --- a/spynepi/util/pypi.py +++ b/spynepi/util/pypi.py @@ -16,7 +16,14 @@ __docformat__ = 'restructuredtext' +import logging +logger = logging.getLogger(__name__) + +import os +import time +import urllib import re + import platform if platform.python_version().startswith('2'): import xmlrpclib @@ -26,16 +33,11 @@ import xmlrpc.client as xmlrpclib import pickle import urllib.request as urllib2 -import os -import time -import logging -import urllib - -from yolk.utils import get_yolk_dir XML_RPC_SERVER = 'http://pypi.python.org/pypi' + class addinfourl(urllib2.addinfourl): """ Replacement addinfourl class compatible with python-2.7's xmlrpclib @@ -60,8 +62,7 @@ def getheaders(self): class ProxyTransport(xmlrpclib.Transport): - """ - Provides an XMl-RPC transport routing via a http proxy. + """Provides an XMl-RPC transport routing via a http proxy. This is done by using urllib2, which in turn uses the environment varable http_proxy and whatever else it is built to use (e.g. the @@ -83,6 +84,7 @@ def request(self, host, handler, request_body, verbose): url = 'http://' + host + handler request = urllib2.Request(url) request.add_data(request_body) + logger.debug("request_body: %r", request_body) # Note: 'Host' and 'Content-Length' are added automatically request.add_header('User-Agent', self.user_agent) request.add_header('Content-Type', 'text/xml') @@ -118,72 +120,27 @@ def check_proxy_setting(): class CheeseShop(object): - """Interface to Python Package Index""" - def __init__(self, debug=False, no_cache=False, yolk_dir=None): - self.debug = debug - self.no_cache = no_cache - if yolk_dir: - self.yolk_dir = yolk_dir - else: - self.yolk_dir = get_yolk_dir() - self.xmlrpc = self.get_xmlrpc_server() - self.pkg_cache_file = self.get_pkg_cache_file() - self.last_sync_file = self.get_last_sync_file() - self.pkg_list = None - self.logger = logging.getLogger("yolk") - self.get_cache() - - def get_cache(self): - """ - Get a package name list from disk cache or PyPI - """ - #This is used by external programs that import `CheeseShop` and don't - #want a cache file written to ~/.pypi and query PyPI every time. - if self.no_cache: - self.pkg_list = self.list_packages() - return - - if not os.path.exists(self.yolk_dir): - os.mkdir(self.yolk_dir) - if os.path.exists(self.pkg_cache_file): - self.pkg_list = self.query_cached_package_list() - else: - self.logger.debug("DEBUG: Fetching package list cache from PyPi...") - self.fetch_pkg_list() - - def get_last_sync_file(self): - """ - Get the last time in seconds since The Epoc since the last pkg list sync - """ - return os.path.abspath(self.yolk_dir + "/last_sync") - - def get_xmlrpc_server(self): - """ - Returns PyPI's XML-RPC server instance - """ + def __init__(self): check_proxy_setting() - if os.environ.has_key('XMLRPC_DEBUG'): + + if logger.level == logging.DEBUG: debug = 1 else: debug = 0 + try: - return xmlrpclib.Server(XML_RPC_SERVER, transport=ProxyTransport(), verbose=debug) + self.xmlrpc = xmlrpclib.Server(XML_RPC_SERVER, + transport=ProxyTransport(), verbose=debug) except IOError: - self.logger("ERROR: Can't connect to XML-RPC server: %s" \ + logger("ERROR: Can't connect to XML-RPC server: %s" \ % XML_RPC_SERVER) - def get_pkg_cache_file(self): - """ - Returns filename of pkg cache - """ - return os.path.abspath('%s/pkg_list.pkl' % self.yolk_dir) - def query_versions_pypi(self, package_name): """Fetch list of available versions for a package from The CheeseShop""" if not package_name in self.pkg_list: - self.logger.debug("Package %s not in cache, querying PyPI..." \ + logger.debug("Package %s not in cache, querying PyPI..." \ % package_name) self.fetch_pkg_list() #I have to set version=[] for edge cases like "Magic file extensions" @@ -193,21 +150,15 @@ def query_versions_pypi(self, package_name): for pypi_pkg in self.pkg_list: if pypi_pkg.lower() == package_name.lower(): if self.debug: - self.logger.debug("DEBUG: %s" % package_name) + logger.debug("DEBUG: %s" % package_name) versions = self.package_releases(pypi_pkg) package_name = pypi_pkg break return (package_name, versions) - def query_cached_package_list(self): - """Return list of pickled package names from PYPI""" - if self.debug: - self.logger.debug("DEBUG: reading pickled cache file") - return cPickle.load(open(self.pkg_cache_file, "r")) - def fetch_pkg_list(self): """Fetch and cache master list of package names from PYPI""" - self.logger.debug("DEBUG: Fetching package name list from PyPI") + logger.debug("DEBUG: Fetching package name list from PyPI") package_list = self.list_packages() cPickle.dump(package_list, open(self.pkg_cache_file, "w")) self.pkg_list = package_list @@ -245,7 +196,7 @@ def release_data(self, package_name, version): def package_releases(self, package_name): """Query PYPI via XMLRPC interface for a pkg's available versions""" if self.debug: - self.logger.debug("DEBUG: querying PyPI for versions of " \ + logger.debug("DEBUG: querying PyPI for versions of " \ + package_name) return self.xmlrpc.package_releases(package_name) From 318d68d26659a8115bcc0ba2c223232337f07508 Mon Sep 17 00:00:00 2001 From: Burak Arslan Date: Wed, 26 Dec 2012 19:11:29 +0200 Subject: [PATCH 04/10] make package caching work for the first time. --- spynepi/entity/html.py | 56 ++++++--- spynepi/main.py | 1 + spynepi/util/pypi.py | 277 ----------------------------------------- 3 files changed, 41 insertions(+), 293 deletions(-) delete mode 100644 spynepi/util/pypi.py diff --git a/spynepi/entity/html.py b/spynepi/entity/html.py index 008b533..65a71d3 100644 --- a/spynepi/entity/html.py +++ b/spynepi/entity/html.py @@ -19,15 +19,22 @@ # MA 02110-1301, USA. # +import logging +logger = logging.getLogger(__name__) + import os +import shutil +import tempfile import subprocess from lxml import html from sqlalchemy import sql +from spyne.error import ValidationError + from pkg_resources import resource_filename -from setuptools.command.easy_install import main as easy_install +from sqlalchemy.orm.exc import NoResultFound from spyne.decorator import rpc from spyne.error import RequestNotAllowed @@ -63,22 +70,35 @@ def index(ctx): -def cache_packages(project_name): - path = os.path.join(FILES_PATH,"files","tmp") - if not os.path.exists(path): - os.makedirs(path) +def cache_package(spec): + from glob import glob + from setuptools.command.easy_install import main as easy_install - easy_install(["--user", "-U", "--build-directory", path, project_name]) - dpath = os.path.join(path,project_name) - dpath = os.path.abspath(dpath) + path = tempfile.mkdtemp('.spynepi') + easy_install(["--user", "-U", "--editable", "--build-directory", + path, spec]) - if not dpath.startswith(path): - # This request tried to read arbitrary data from the filesystem - raise RequestNotAllowed(repr([project_name,])) + # plagiarized from setuptools + try: + setups = glob(os.path.join(path, '*', 'setup.py')) + if not setups: + raise ValidationError( + "Couldn't find a setup script in %r editable distribution: %r" % + (spec, os.path.join(path,'*')) + ) + if len(setups)>1: + raise ValidationError( + "Multiple setup scripts in found in %r editable distribution: %r" % + (spec, setups) + ) - command = ["python", "setup.py", "register", "-r", REPO_NAME, "sdist", - "upload", "-r", REPO_NAME] - subprocess.call(command, cwd=dpath) + command = ["python", "setup.py", "register", "-r", REPO_NAME, "sdist", + "upload", "-r", REPO_NAME] + logger.info('calling %r', command) + subprocess.call(command, cwd=os.path.dirname(setups[0])) + + finally: + shutil.rmtree(path) class HtmlService(ServiceBase): @@ -88,15 +108,19 @@ class HtmlService(ServiceBase): HttpPattern("//"), HttpPattern("///"), ]) - def download_html(ctx,project_name,version): + def download_html(ctx, project_name, version): ctx.transport.mime_type = "text/html" - ctx.udc.session.query(Package).filter_by( + try: + ctx.udc.session.query(Package).filter_by( package_name=project_name).one() + except NoResultFound: + cache_package(project_name) download = HtmlPage(TPL_DOWNLOAD) download.title = project_name + if version: release = ctx.udc.session.query(Release).join(Package).filter( sql.and_( diff --git a/spynepi/main.py b/spynepi/main.py index 733e4f7..77fe522 100644 --- a/spynepi/main.py +++ b/spynepi/main.py @@ -72,6 +72,7 @@ def call_wrapper(self, ctx): return Application.call_wrapper(self, ctx) except NoResultFound, e: + logger.exception(e) ctx.out_string = ["Resource not found"] raise ResourceNotFoundError() # Return HTTP 404 diff --git a/spynepi/util/pypi.py b/spynepi/util/pypi.py deleted file mode 100644 index f4bd453..0000000 --- a/spynepi/util/pypi.py +++ /dev/null @@ -1,277 +0,0 @@ - -""" - -pypi.py -======= - -Desc: Library for getting information about Python packages by querying - The CheeseShop (PYPI a.k.a. Python Package Index). - - -Author: Rob Cakebread - -License : BSD (See COPYING) - -""" - -__docformat__ = 'restructuredtext' - -import logging -logger = logging.getLogger(__name__) - -import os -import time -import urllib -import re - -import platform -if platform.python_version().startswith('2'): - import xmlrpclib - import cPickle - import urllib2 -else: - import xmlrpc.client as xmlrpclib - import pickle - import urllib.request as urllib2 - - -XML_RPC_SERVER = 'http://pypi.python.org/pypi' - - -class addinfourl(urllib2.addinfourl): - """ - Replacement addinfourl class compatible with python-2.7's xmlrpclib - - In python-2.7, xmlrpclib expects that the response object that it receives - has a getheader method. httplib.HTTPResponse provides this but - urllib2.addinfourl does not. Add the necessary functions here, ported to - use the internal data structures of addinfourl. - """ - - def getheader(self, name, default=None): - if self.headers is None: - raise httplib.ResponseNotReady() - return self.headers.getheader(name, default) - - def getheaders(self): - if self.headers is None: - raise httplib.ResponseNotReady() - return self.headers.items() - -urllib2.addinfourl = addinfourl - - -class ProxyTransport(xmlrpclib.Transport): - """Provides an XMl-RPC transport routing via a http proxy. - - This is done by using urllib2, which in turn uses the environment - varable http_proxy and whatever else it is built to use (e.g. the - windows registry). - - NOTE: the environment variable http_proxy should be set correctly. - See check_proxy_setting() below. - - Written from scratch but inspired by xmlrpc_urllib_transport.py - file from http://starship.python.net/crew/jjkunce/ by jjk. - - A. Ellerton 2006-07-06 - """ - - def request(self, host, handler, request_body, verbose): - '''Send xml-rpc request using proxy''' - #We get a traceback if we don't have this attribute: - self.verbose = verbose - url = 'http://' + host + handler - request = urllib2.Request(url) - request.add_data(request_body) - logger.debug("request_body: %r", request_body) - # Note: 'Host' and 'Content-Length' are added automatically - request.add_header('User-Agent', self.user_agent) - request.add_header('Content-Type', 'text/xml') - proxy_handler = urllib2.ProxyHandler() - opener = urllib2.build_opener(proxy_handler) - fhandle = opener.open(request) - return(self.parse_response(fhandle)) - - -def check_proxy_setting(): - """ - If the environmental variable 'HTTP_PROXY' is set, it will most likely be - in one of these forms: - - proxyhost:8080 - http://proxyhost:8080 - - urlllib2 requires the proxy URL to start with 'http://' - This routine does that, and returns the transport for xmlrpc. - """ - try: - http_proxy = os.environ['HTTP_PROXY'] - except KeyError: - return - - if not http_proxy.startswith('http://'): - match = re.match('(http://)?([-_\.A-Za-z]+):(\d+)', http_proxy) - #if not match: - # raise Exception('Proxy format not recognised: [%s]' % http_proxy) - os.environ['HTTP_PROXY'] = 'http://%s:%s' % (match.group(2), - match.group(3)) - return - - -class CheeseShop(object): - """Interface to Python Package Index""" - - def __init__(self): - check_proxy_setting() - - if logger.level == logging.DEBUG: - debug = 1 - else: - debug = 0 - - try: - self.xmlrpc = xmlrpclib.Server(XML_RPC_SERVER, - transport=ProxyTransport(), verbose=debug) - except IOError: - logger("ERROR: Can't connect to XML-RPC server: %s" \ - % XML_RPC_SERVER) - - def query_versions_pypi(self, package_name): - """Fetch list of available versions for a package from The CheeseShop""" - if not package_name in self.pkg_list: - logger.debug("Package %s not in cache, querying PyPI..." \ - % package_name) - self.fetch_pkg_list() - #I have to set version=[] for edge cases like "Magic file extensions" - #but I'm not sure why this happens. It's included with Python or - #because it has a space in it's name? - versions = [] - for pypi_pkg in self.pkg_list: - if pypi_pkg.lower() == package_name.lower(): - if self.debug: - logger.debug("DEBUG: %s" % package_name) - versions = self.package_releases(pypi_pkg) - package_name = pypi_pkg - break - return (package_name, versions) - - def fetch_pkg_list(self): - """Fetch and cache master list of package names from PYPI""" - logger.debug("DEBUG: Fetching package name list from PyPI") - package_list = self.list_packages() - cPickle.dump(package_list, open(self.pkg_cache_file, "w")) - self.pkg_list = package_list - - def search(self, spec, operator): - '''Query PYPI via XMLRPC interface using search spec''' - return self.xmlrpc.search(spec, operator.lower()) - - def changelog(self, hours): - '''Query PYPI via XMLRPC interface using search spec''' - return self.xmlrpc.changelog(get_seconds(hours)) - - def updated_releases(self, hours): - '''Query PYPI via XMLRPC interface using search spec''' - return self.xmlrpc.updated_releases(get_seconds(hours)) - - def list_packages(self): - """Query PYPI via XMLRPC interface for a a list of all package names""" - return self.xmlrpc.list_packages() - - def release_urls(self, package_name, version): - """Query PYPI via XMLRPC interface for a pkg's available versions""" - - return self.xmlrpc.release_urls(package_name, version) - - def release_data(self, package_name, version): - """Query PYPI via XMLRPC interface for a pkg's metadata""" - try: - return self.xmlrpc.release_data(package_name, version) - except xmlrpclib.Fault: - #XXX Raises xmlrpclib.Fault if you give non-existant version - #Could this be server bug? - return - - def package_releases(self, package_name): - """Query PYPI via XMLRPC interface for a pkg's available versions""" - if self.debug: - logger.debug("DEBUG: querying PyPI for versions of " \ - + package_name) - return self.xmlrpc.package_releases(package_name) - - def get_download_urls(self, package_name, version="", pkg_type="all"): - """Query PyPI for pkg download URI for a packge""" - - if version: - versions = [version] - else: - - #If they don't specify version, show em all. - - (package_name, versions) = self.query_versions_pypi(package_name) - - all_urls = [] - for ver in versions: - metadata = self.release_data(package_name, ver) - for urls in self.release_urls(package_name, ver): - if pkg_type == "source" and urls['packagetype'] == "sdist": - all_urls.append(urls['url']) - elif pkg_type == "egg" and \ - urls['packagetype'].startswith("bdist"): - all_urls.append(urls['url']) - elif pkg_type == "all": - #All - all_urls.append(urls['url']) - - #Try the package's metadata directly in case there's nothing - #returned by XML-RPC's release_urls() - if metadata and metadata.has_key('download_url') and \ - metadata['download_url'] != "UNKNOWN" and \ - metadata['download_url'] != None: - if metadata['download_url'] not in all_urls: - if pkg_type != "all": - url = filter_url(pkg_type, metadata['download_url']) - if url: - all_urls.append(url) - return all_urls - -def filter_url(pkg_type, url): - """ - Returns URL of specified file type - 'source', 'egg', or 'all' - """ - bad_stuff = ["?modtime", "#md5="] - for junk in bad_stuff: - if junk in url: - url = url.split(junk)[0] - break - - #pkg_spec==dev (svn) - if url.endswith("-dev"): - url = url.split("#egg=")[0] - - if pkg_type == "all": - return url - - elif pkg_type == "source": - valid_source_types = [".tgz", ".tar.gz", ".zip", ".tbz2", ".tar.bz2"] - for extension in valid_source_types: - if url.lower().endswith(extension): - return url - - elif pkg_type == "egg": - if url.lower().endswith(".egg"): - return url - -def get_seconds(hours): - """ - Get number of seconds since epoch from now minus `hours` - - @param hours: Number of `hours` back in time we are checking - @type hours: int - - Return integer for number of seconds for now minus hours - - """ - return int(time.time() - (60 * 60) * hours) From 396d32d9e6d9aa74b8a5618362ed504256aa0802 Mon Sep 17 00:00:00 2001 From: Burak Arslan Date: Thu, 27 Dec 2012 15:45:25 +0200 Subject: [PATCH 05/10] raise 404 for upload before first register. --- spynepi/entity/root.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spynepi/entity/root.py b/spynepi/entity/root.py index 8bfca25..555bd05 100644 --- a/spynepi/entity/root.py +++ b/spynepi/entity/root.py @@ -29,6 +29,7 @@ from spyne.decorator import rpc from spyne.error import ArgumentError +from spyne.error import ResourceNotFoundError from spyne.model.primitive import String from spyne.model.primitive import Unicode from spyne.service import ServiceBase @@ -133,6 +134,8 @@ def package_content(): Release.release_version == version) ).first() + if rel is None: + raise ResourceNotFoundError(name) rel.distributions.append(generate_dist()) package_content() From 732842e4820960e7b08c0f8b5be7737795d1d2b7 Mon Sep 17 00:00:00 2001 From: Burak Arslan Date: Thu, 27 Dec 2012 15:45:34 +0200 Subject: [PATCH 06/10] whitespace --- spynepi/main.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/spynepi/main.py b/spynepi/main.py index 77fe522..ab90dc1 100644 --- a/spynepi/main.py +++ b/spynepi/main.py @@ -83,14 +83,14 @@ def main(connection_string=DB_CONNECTION_STRING): logging.getLogger('spyne.protocol.xml').setLevel(logging.DEBUG) #logging.getLogger('sqlalchemy.engine.base.Engine').setLevel(logging.DEBUG) - index_app = MyApplication([RootService, IndexService],"http://usefulinc.com/ns/doap#", - in_protocol=HttpRpc(), out_protocol=HtmlTable()) + index_app = MyApplication([RootService, IndexService], "http://usefulinc.com/ns/doap#", + in_protocol=HttpRpc(), out_protocol=HtmlTable()) - rdf_app = MyApplication([RdfService],"http://usefulinc.com/ns/doap#", - in_protocol=HttpRpc(), out_protocol=XmlDocument()) + rdf_app = MyApplication([RdfService], "http://usefulinc.com/ns/doap#", + in_protocol=HttpRpc(), out_protocol=XmlDocument()) html_app = MyApplication([HtmlService],"http://usefulinc.com/ns/doap#", - in_protocol=HttpRpc(), out_protocol=HttpRpc()) + in_protocol=HttpRpc(), out_protocol=HttpRpc()) db_handle = init_database(connection_string) @@ -108,6 +108,7 @@ def _on_method_call(ctx): # this is called once all data is sent to the client. def _on_method_return_object(ctx): ctx.udc.session.commit() + def _on_wsgi_close(ctx): if ctx.udc is not None: ctx.udc.close() @@ -123,7 +124,6 @@ def _on_wsgi_close(ctx): for a in wsgi_index,wsgi_rdf,wsgi_html: a.event_manager.add_listener('wsgi_close', _on_wsgi_close) - url_map = Map([Rule("/", endpoint=wsgi_index), Rule("/", endpoint=wsgi_html), Rule("//", endpoint=wsgi_html), From b20b6576c7761adcc2e8b4192b7ed5080a2bac71 Mon Sep 17 00:00:00 2001 From: Burak Arslan Date: Thu, 27 Dec 2012 16:23:33 +0200 Subject: [PATCH 07/10] generate own .pypirc --- spynepi/entity/html.py | 43 ++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 41 insertions(+), 2 deletions(-) diff --git a/spynepi/entity/html.py b/spynepi/entity/html.py index 65a71d3..935bc94 100644 --- a/spynepi/entity/html.py +++ b/spynepi/entity/html.py @@ -31,6 +31,7 @@ from sqlalchemy import sql from spyne.error import ValidationError +from spyne.util import reconstruct_url from pkg_resources import resource_filename @@ -69,15 +70,53 @@ def index(ctx): ) for package in ctx.udc.session.query(Package)] - -def cache_package(spec): +def cache_package(spec, own_url): from glob import glob from setuptools.command.easy_install import main as easy_install + import ConfigParser path = tempfile.mkdtemp('.spynepi') easy_install(["--user", "-U", "--editable", "--build-directory", path, spec]) + if os.environ.has_key('HOME'): + rc = os.path.join(os.environ['HOME'], '.pypirc') + config = ConfigParser.ConfigParser() + + if os.path.exists(rc): + config.read(rc) + + try: + config.add_section(REPO_NAME) + + config.set(REPO_NAME, 'repository', own_url) + config.set(REPO_NAME, 'username', 'x') + config.set(REPO_NAME, 'password', 'y') + + except ConfigParser.DuplicateSectionError: + pass + + try: + config.add_section('distutils') + except ConfigParser.DuplicateSectionError: + pass + + try: + index_servers = config.get('distutils', 'index-servers') + index_servers = index_servers.split('\n') + if 'spynepi' not in index_servers: + index_servers.append(REPO_NAME) + + except ConfigParser.NoOptionError: + index_servers = [REPO_NAME] + + config.set('distutils', 'index-servers', '\n'.join(index_servers)) + + config.write(open(rc,'w')) + + else: # FIXME: ??? No idea. Hopefully setuptools knows better. + pass # raise NotImplementedError("$HOME not defined, .pypirc not found.") + # plagiarized from setuptools try: setups = glob(os.path.join(path, '*', 'setup.py')) From 1a40a5c95aaa89c0944b6331b83c831332bdb7f6 Mon Sep 17 00:00:00 2001 From: Burak Arslan Date: Thu, 27 Dec 2012 16:23:57 +0200 Subject: [PATCH 08/10] parse setup.py instead of calling another interpreter with subprocess --- spynepi/entity/html.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/spynepi/entity/html.py b/spynepi/entity/html.py index 935bc94..118ccff 100644 --- a/spynepi/entity/html.py +++ b/spynepi/entity/html.py @@ -25,7 +25,6 @@ import os import shutil import tempfile -import subprocess from lxml import html from sqlalchemy import sql @@ -73,6 +72,7 @@ def index(ctx): def cache_package(spec, own_url): from glob import glob from setuptools.command.easy_install import main as easy_install + from distutils.core import run_setup import ConfigParser path = tempfile.mkdtemp('.spynepi') @@ -117,31 +117,40 @@ def cache_package(spec, own_url): else: # FIXME: ??? No idea. Hopefully setuptools knows better. pass # raise NotImplementedError("$HOME not defined, .pypirc not found.") - # plagiarized from setuptools try: + # plagiarized from setuptools setups = glob(os.path.join(path, '*', 'setup.py')) if not setups: raise ValidationError( "Couldn't find a setup script in %r editable distribution: %r" % (spec, os.path.join(path,'*')) ) + if len(setups)>1: raise ValidationError( "Multiple setup scripts in found in %r editable distribution: %r" % (spec, setups) ) - command = ["python", "setup.py", "register", "-r", REPO_NAME, "sdist", - "upload", "-r", REPO_NAME] - logger.info('calling %r', command) - subprocess.call(command, cwd=os.path.dirname(setups[0])) + lib_dir =os.path.dirname(setups[0]) + os.chdir(lib_dir) + dist = run_setup(setups[0]) + dist.commands = ['register', 'sdist', 'upload'] + dist.command_options = { + 'register': {'repository': ('command line', 'spynepi')}, + 'upload': {'repository': ('command line', 'spynepi')}, + 'sdist': {}, + } + + dist.run_commands() + finally: shutil.rmtree(path) class HtmlService(ServiceBase): - @rpc(Unicode, Unicode,_returns=Unicode, _patterns=[ + @rpc(Unicode, Unicode, _returns=Unicode, _patterns=[ HttpPattern("/"), HttpPattern("//"), HttpPattern("//"), @@ -154,12 +163,12 @@ def download_html(ctx, project_name, version): ctx.udc.session.query(Package).filter_by( package_name=project_name).one() except NoResultFound: - cache_package(project_name) + cache_package(project_name, reconstruct_url(ctx.transport.req_env, + path=False, query_string=False)) download = HtmlPage(TPL_DOWNLOAD) download.title = project_name - if version: release = ctx.udc.session.query(Release).join(Package).filter( sql.and_( From 7fad2221ef93cf0494482970237a66c1e5433714 Mon Sep 17 00:00:00 2001 From: Burak Arslan Date: Thu, 27 Dec 2012 16:25:49 +0200 Subject: [PATCH 09/10] Add changelog --- CHANGELOG | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 CHANGELOG diff --git a/CHANGELOG b/CHANGELOG new file mode 100644 index 0000000..b7142f0 --- /dev/null +++ b/CHANGELOG @@ -0,0 +1,14 @@ +0.3 +=== +* Add package caching + + +0.2 +=== +* Add support for Spyne 2.9 APIs. + + +0.1 +=== + +* First working version From 76976aa86bd8cdedf9407260059643ad3c6e4170 Mon Sep 17 00:00:00 2001 From: Burak Arslan Date: Thu, 27 Dec 2012 16:27:00 +0200 Subject: [PATCH 10/10] version bump --- setup.py | 2 +- spynepi/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f82ad2f..1d99414 100755 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ license='GPL', install_requires=[ - "spyne<2.99", "sqlalchemy<0.8", "werkzeug", "twisted", + "spyne>=2.10", "sqlalchemy<0.8", "werkzeug", "twisted", ], include_package_data=True, entry_points = { diff --git a/spynepi/__init__.py b/spynepi/__init__.py index bf9bb1d..b4b5c10 100644 --- a/spynepi/__init__.py +++ b/spynepi/__init__.py @@ -19,5 +19,5 @@ # MA 02110-1301, USA. # -__version__ = '0.1' +__version__ = '0.3'