Merge pull request ckan#8 from clients/dga-uat

UAT merge for CKAN 2.7.4 deployment

.travis.yml

@@ -1,14 +1,22 @@
 language: python
+
+# the new trusty images of Travis cause build errors with psycopg2, see https://github.com/travis-ci/travis-ci/issues/8897
+dist: trusty
+group: deprecated-2017Q4
+
 python:
- - "2.6"
  - "2.7"
 env:
- - PGVERSION=9.1 CKANVERSION=master
- - PGVERSION=9.1 CKANVERSION=2.3
- - PGVERSION=9.1 CKANVERSION=2.4.0
+ - CKANVERSION=master
+ - CKANVERSION=2.7
+ - CKANVERSION=2.6
+ - CKANVERSION=2.5
+ - CKANVERSION=2.4
+ - CKANVERSION=2.3
 install:
  - bash bin/travis-build.bash
- - pip install coveralls
+services:
+ -postgresql
 script: sh bin/travis-run.sh
-after_success:
- - coveralls
+after_success: coveralls
+sudo: required

MANIFEST.in

@@ -1,2 +1,3 @@
 include README.rst
-recursive-include ckanext/showcase *.html *.json *.js *.less *.css
+recursive-include ckanext/showcase *.html *.json *.js *.less *.css
+recursive-include ckanext/showcase/i18n *

README.rst

@@ -5,12 +5,12 @@
 .. image:: https://travis-ci.org/ckan/ckanext-showcase.svg?branch=master
  :target: https://travis-ci.org/ckan/ckanext-showcase
 
-.. image:: https://coveralls.io/repos/ckan/ckanext-showcase/badge.png?branch=master
- :target: https://coveralls.io/r/ckan/ckanext-showcase?branch=master
+.. image:: https://coveralls.io/repos/ckan/ckanext-showcase/badge.svg
+ :target: https://coveralls.io/r/ckan/ckanext-showcase
 
-=============
+================
 ckanext-showcase
-=============
+================
 
 Showcase and link to datasets in use. Datasets used in an app, website or
 visualization, or featured in an article, report or blog post can be showcased
@@ -30,9 +30,13 @@ ckanext-showcase is intended to be a more powerful replacement for the
 Requirements
 ------------
 
-Current status: Beta
 
-Compatible with CKAN 2.3.
+Compatible with CKAN 2.3+.
+
+N.B. The ``migrate`` command, detailed below, requires the Related Item models
+and actions, which have been removed in CKAN 2.6. If you wish to migrate your
+Related Items, please first upgrade CKAN to 2.5, do the migration, then
+continue upgrading to CKAN 2.6+.
 
 
 ------------
@@ -135,7 +139,11 @@ From the ``ckanext-showcase`` directory::
  paster showcase migrate -c {path to production.ini}
 
 Note that each Related Item must have a unique title before migration can
-proceed.
+proceed. If you prefer resolving duplicates as showcases, you can use the --allow-duplicates
+option to migrate them anyways. Duplicate Relations will be created as
+'duplicate_' + original_related_title + '_' + related_id
+
+ paster showcase migrate -c {path to production.ini} --allow-duplicates
 
 The Related Item property ``type`` will become a Showcase tag. The Related Item
 properties ``created``, ``owner_id``, ``view_count``, and ``featured`` have no
@@ -157,9 +165,9 @@ coverage installed in your virtualenv (``pip install coverage``) then run::
  nosetests --ckan --nologcapture --with-pylons=test.ini --with-coverage --cover-package=ckanext.showcase --cover-inclusive --cover-erase --cover-tests
 
 
----------------------------------
+------------------------------------
 Registering ckanext-showcase on PyPI
----------------------------------
+------------------------------------
 
 ckanext-showcase should be availabe on PyPI as
 https://pypi.python.org/pypi/ckanext-showcase. If that link doesn't work, then
@@ -186,9 +194,9 @@ steps:
  git push --tags
 
 
-----------------------------------------
+-------------------------------------------
 Releasing a New Version of ckanext-showcase
-----------------------------------------
+-------------------------------------------
 
 ckanext-showcase is availabe on PyPI as https://pypi.python.org/pypi/ckanext-showcase.
 To publish a new version to PyPI follow these steps:
@@ -211,3 +219,22 @@ To publish a new version to PyPI follow these steps:
 
  git tag 0.0.2
  git push --tags
+
+
+-------------------------------------------
+i18n
+-------------------------------------------
+
+See: "Internationalizing strings in extensions" : http://docs.ckan.org/en/latest/extensions/translating-extensions.html
+
+1. Install babel
+
+ pip install Babel
+
+2. Init Catalog for your language
+
+ python setup.py init_catalog -l es
+
+3. Compile your language catalog ( You can force pybabel compile to compile messages marked as fuzzy with the -f)
+
+ python setup.py compile_catalog -f -l es

bin/travis-build.bash

@@ -5,16 +5,18 @@ echo "This is travis-build.bash..."
 
 echo "Installing the packages that CKAN requires..."
 sudo apt-get update -qq
-sudo apt-get install postgresql-$PGVERSION solr-jetty libcommons-fileupload-java:amd64=1.2.2-1
+sudo apt-get install solr-jetty libcommons-fileupload-java
 
 echo "Installing CKAN and its Python dependencies..."
 git clone https://github.com/ckan/ckan
 cd ckan
-if [ $ CKANVERSION == '2.3']
+if [ $CKANVERSION == 'master' ]
 then
- git checkout release-v2.3
+ echo "CKAN version: master"
 else
- git checkout master
+ CKAN_TAG=$(git tag | grep ^ckan-$CKANVERSION | sort --version-sort | tail -n 1)
+ git checkout $CKAN_TAG
+ echo "CKAN version: ${CKAN_TAG#ckan-}"
 fi
 python setup.py develop
 pip install -r requirements.txt --allow-all-external
@@ -30,12 +32,12 @@ cd ckan
 paster db init -c test-core.ini
 cd -
 
-echo "Installing ckanext-ckanext-showcase and its requirements..."
+echo "Installing ckanext-showcase and its requirements..."
 python setup.py develop
 pip install -r dev-requirements.txt
 
 echo "Moving test.ini into a subdir..."
 mkdir subdir
-mv test.ini subdir
+mv test-travis.ini subdir
 
 echo "travis-build.bash is done."

bin/travis-run.sh

@@ -3,4 +3,4 @@
 echo "NO_START=0\nJETTY_HOST=127.0.0.1\nJETTY_PORT=8983\nJAVA_HOME=$JAVA_HOME" | sudo tee /etc/default/jetty
 sudo cp ckan/ckan/config/solr/schema.xml /etc/solr/conf/schema.xml
 sudo service jetty restart
-nosetests --ckan --nologcapture --with-pylons=subdir/test.ini --with-coverage --cover-package=ckanext.showcase --cover-inclusive --cover-erase --cover-tests
+nosetests --ckan --nologcapture --with-pylons=subdir/test-travis.ini --with-coverage --cover-package=ckanext.showcase --cover-inclusive --cover-erase --cover-tests

ckanext/showcase/commands/migrate.py

@@ -17,11 +17,23 @@ class MigrationCommand(CkanCommand):
  paster showcase migrate -c <path to config file>
  - Migrate Related Items to Showcases
 
+ paster showcase migrate -c <path to config file> [--allow-duplicates]
+ - Migrate Related Items to Showcases and allow duplicates
+
  Must be run from the ckanext-showcase directory.
  '''
  summary = __doc__.split('\n')[0]
  usage = __doc__
 
+ def __init__(self,name):
+ super(CkanCommand, self).__init__(name)
+
+ self.parser.add_option('--allow-duplicates', dest='allow_duplicates',
+ default=False, help='''Use this option to allow
+ related items with duplicate titles to be migrated.
+ Duplicate showcases will be created as
+ 'duplicate_<related-name>_<related-id>'.''', action='store_true')
+
  def command(self):
  '''
  Parse command line arguments and call appropriate method.
@@ -44,14 +56,17 @@ def migrate(self):
  '''
 
  '''
+ # determine whether migration should allow duplicates
+ allow_duplicates = self.options.allow_duplicates
+
  related_items = get_action('related_list')(data_dict={})
 
  # preflight:
  # related items must have unique titles before migration
  related_titles = [i['title'] for i in related_items]
  # make a list of duplicate titles
  duplicate_titles = self._find_duplicates(related_titles)
- if duplicate_titles:
+ if duplicate_titles and allow_duplicates == False:
  print(
  """All Related Items must have unique titles before migration. The following
 Related Item titles are used more than once and need to be corrected before
@@ -69,10 +84,11 @@ def migrate(self):
  print('Showcase for Related Item "{0}" already exists.'.format(
  normalized_title))
  else:
+ showcase_title = self._gen_new_title(related.get('title'), related['id'])
  data_dict = {
  'original_related_item_id': related.get('id'),
- 'title': related.get('title'),
- 'name': munge_title_to_name(related.get('title')),
+ 'title': showcase_title,
+ 'name': munge_title_to_name(showcase_title),
  'notes': related.get('description'),
  'image_url': related.get('image_url'),
  'url': related.get('url'),
@@ -120,3 +136,12 @@ def _find_duplicates(self, lst):
  ['two', 'three']
  '''
  return list(set(x for x in lst if lst.count(x) >= 2))
+
+ def _gen_new_title(self, title, related_id):
+ name = munge_title_to_name(title)
+ pkg_obj = model.Session.query(model.Package).filter_by(name=name).first()
+ if pkg_obj:
+ title.replace('duplicate_', '')
+ return 'duplicate_' + title + '_' + related_id
+ else:
+ return title

ckanext/showcase/controller.py

@@ -5,12 +5,11 @@
 
 from ckan.plugins import toolkit as tk
 import ckan.model as model
-import ckan.lib.base as base
 import ckan.lib.helpers as h
 import ckan.lib.navl.dictization_functions as dict_fns
 import ckan.logic as logic
 import ckan.plugins as p
-from ckan.common import OrderedDict, g, ungettext
+from ckan.common import OrderedDict, ungettext
 from ckan.controllers.package import (PackageController,
  url_with_params,
  _encode_params)
@@ -23,7 +22,7 @@
 request = tk.request
 render = tk.render
 abort = tk.abort
-redirect = base.redirect
+redirect = tk.redirect_to
 NotFound = tk.ObjectNotFound
 ValidationError = tk.ValidationError
 check_access = tk.check_access
@@ -358,9 +357,13 @@ def _add_dataset_search(self, showcase_id, showcase_name):
  # unicode format (decoded from utf8)
  q = c.q = request.params.get('q', u'')
  c.query_error = False
- page = self._get_page_number(request.params)
+ try:
+ page = self._get_page_number(request.params)
+ except AttributeError:
+ # in CKAN >= 2.5 _get_page_number has been moved
+ page = h.get_page_number(request.params)
 
- limit = g.datasets_per_page
+ limit = int(config.get('ckan.datasets_per_page', 20))
 
  # most search operations should reset the page counter:
  params_nopage = [(k, v) for k, v in request.params.items()
@@ -466,7 +469,15 @@ def pager_url(q=None, page=None):
  'license_id': _('Licenses'),
  }
 
- for facet in g.facets:
+ # for CKAN-Versions that do not provide the facets-method from
+ # helper-context, import facets from ckan.common
+ if hasattr(h, 'facets'):
+ current_facets = h.facets()
+ else:
+ from ckan.common import g
+ current_facets = g.facets
+
+ for facet in current_facets:
  if facet in default_facet_titles:
  facets[facet] = default_facet_titles[facet]
  else:
@@ -511,7 +522,7 @@ def pager_url(q=None, page=None):
  for facet in c.search_facets.keys():
  try:
  limit = int(request.params.get('_%s_limit' % facet,
- g.facets_default_number))
+ int(config.get('search.facets.default', 10))))
  except ValueError:
  abort(400, _("Parameter '{parameter_name}' is not an integer").format(
  parameter_name='_%s_limit' % facet

ckanext/showcase/i18n/__init__.py

@@ -0,0 +1,7 @@
+# this is a namespace package
+try:
+ import pkg_resources
+ pkg_resources.declare_namespace(__name__)
+except ImportError:
+ import pkgutil
+ __path__ = pkgutil.extend_path(__path__, __name__)