Merge pull request #4 from HBS-HBX/myer/add_basic_test_structure_for_…

…python_2_#2

closes #2 add basic test structure for python 2

.coveragerc

@@ -4,7 +4,7 @@ data_file = .coverage
 source=django_elastic_migrations
 omit =
     test_settings
-    *migrations*
+    */migrations/*
     *admin.py
     *static*
     *templates*

.gitignore

@@ -61,8 +61,5 @@ output/*/index.html
 requirements/private.in
 requirements/private.txt
 
-# tox environment temporary artifacts
-tests/__init__.py
-
 # Development task artifacts
 default.db

.travis.yml

@@ -0,0 +1,42 @@
+# Config file for automatic testing at travis-ci.org
+
+language: python
+python: 3.6
+
+dist: trusty 
+
+matrix:
+  include:
+    - env: TOX_ENV=py27-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
+      python: 2.7
+    - env: TOX_ENV=py27-django19-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
+      python: 2.7
+# TBD support - will be implemented in #5
+#  allow_failures:
+#    - env: TOX_ENV=py27-django19-es62 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
+#      python: 2.7
+#    - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
+#      python: 3.6
+#    - env: TOX_ENV=py36-django18-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt
+#      python: 3.6
+
+before_install:
+  - pip install --upgrade pip
+  # work around https://github.com/travis-ci/travis-ci/issues/8363
+  - pip install codecov
+  - wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add -
+  - echo "deb $ES_APT_URL stable main" | sudo tee -a /etc/apt/sources.list.d/elk.list
+  - sudo apt-get update && sudo apt-get install elasticsearch -y
+  - sudo service elasticsearch start
+
+install: pip install -r requirements/travis.txt
+
+# sleep for elasticsearch
+before_script:
+  - sleep 10
+
+# command to run tests
+script: tox -e $TOX_ENV
+
+after_success:
+  - codecov -e $TOX_ENV

CHANGELOG.rst

@@ -0,0 +1,11 @@
+Changelog
+---------
+
+0.6.0 (2018-08-01)
+~~~~~~~~~~~~~~~~~~
+* Added test structure for py2 - GH #2
+* Renamed default log handler from ``django-elastic-migrations`` to ``django_elastic_migrations``
+
+0.5.3 (2018-07-23)
+~~~~~~~~~~~~~~~~~~
+* First basic release

MANIFEST.in

@@ -2,5 +2,5 @@ include AUTHORS
 include CHANGELOG.rst
 include CONTRIBUTING.rst
 include LICENSE.txt
-include README.rst
+include README.md
 recursive-include django_elastic_migrations *.html *.png *.gif *js *.css *jpg *jpeg *svg *py

Makefile

@@ -1,5 +1,5 @@
 .PHONY: clean coverage help \
-	quality requirements selfcheck test test-all upgrade validate
+	quality requirements selfcheck syncdb load test test-all upgrade validate pylintrc
 
 .DEFAULT_GOAL := help
 
@@ -29,17 +29,14 @@ clean: ## remove generated byte code, coverage reports, and build artifacts
 	rm -fr dist/
 	rm -fr *.egg-info
 
-coverage: clean ## generate and view HTML coverage report
-	py.test --cov-report html
-	$(BROWSER) htmlcov/index.html
-
 upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
 	pip install -q pip-tools
 	pip-compile --upgrade -o requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in
 	pip-compile --upgrade -o requirements/quality.txt requirements/quality.in
 	pip-compile --upgrade -o requirements/test.txt requirements/base.in requirements/test.in
-	# Let tox control the Django version for tests
-	sed '/django==/d' requirements/test.txt > requirements/test.tmp
+	pip-compile --upgrade -o requirements/travis.txt requirements/travis.in
+	# Let tox control the Django and elasticsearch-dsl version for tests
+	sed '/django==/d' requirements/test.txt | sed '/elasticsearch-dsl==/d' | sed '/elasticsearch==/d' > requirements/test.tmp
 	mv requirements/test.tmp requirements/test.txt
 
 quality: ## check coding style with pycodestyle and pylint
@@ -49,17 +46,31 @@ requirements: ## install development environment requirements
 	pip install -qr requirements/dev.txt --exists-action w
 	pip-sync requirements/dev.txt requirements/test.txt
 
-test: clean ## run tests in the current virtualenv
-	py.test
+coverage: clean ## check code coverage quickly with the default Python
+	coverage run ./manage.py test
+	coverage report -m
+	coverage html
+	$(BROWSER) htmlcov/index.html
+
+syncdb: clean ## setup local sqlite db
+	./manage.py migrate --run-syncdb
+
+load: syncdb ## load fixtures into sqlite
+	./manage.py loaddata tests/tests_initial.json
+
+test: syncdb ## run tests in the current virtualenv
+	./manage.py test
 
 diff_cover: test
 	diff-cover coverage.xml
 
 test-all: ## run tests on every supported Python/Django combination
-	tox -e quality
 	tox
 
 validate: quality test ## run tests and quality checks
 
 selfcheck: ## check that the Makefile is well-formed
 	@echo "The Makefile is well-formed."
+
+pylintrc: ## check that the Makefile is well-formed
+	edx_lint write pylintrc

README.md

@@ -1,7 +1,9 @@
 # Django Elastic Migrations
 
 Django Elastic Migrations provides a way to control the deployment of
-multiple Elasticsearch schemas over time. 
+multiple Elasticsearch schemas over time.
+
+[![Build Status](https://travis-ci.com/HBS-HBX/django-elastic-migrations.svg?branch=master)](https://travis-ci.com/HBS-HBX/django-elastic-migrations)
 
 ## Overview
 
@@ -275,13 +277,37 @@ and return the kwargs you would like to customize.
 This project uses `make` to manage the build process. Type `make help`
 to see the available `make` targets.
 
+### Elasticsearch Docker Compose
+
+`docker-compose -f local.yml up`
+
+[See docs/docker_setup for more info](./docs/docker_setup.rst)
+
+
 ### Requirements
+*`make upgrade`* upgrades the dependencies of the requirements to latest
+version. This process also excludes `django` and `elasticsearch-dsl`
+from the `requirements/test.txt` so they can be injected with different
+versions by tox during matrix testing.
 
-Then, `make requirements` runs the pip install. 
+*`make requirements`* runs the pip install.
 
 This project also uses [`pip-tools`](https://github.com/jazzband/pip-tools).
 The `requirements.txt` files are generated and pinned to latest versions 
-with `make upgrade`. 
+with `make upgrade`.
+
+### Running Tests Locally
+
+Run `make test`. To run all tests and quality checks locally,
+run `make test-all`.
+
+To just run linting, `make quality`. Please note that if any of the
+linters return a nonzero code, it will give an `InvocationError` error
+at the end. See [tox's documentation for `InvocationError`](https://tox.readthedocs.io/en/latest/example/general.html#understanding-invocationerror-exit-codes)
+for more information.
+
+We use `edx_lint` to compile `pylintrc`. To update the rules,
+change `pylintrc_tweaks` and run `make pylintrc`.
 
 ### Updating Egg Info
 

codecov.yml

@@ -8,5 +8,7 @@ coverage:
       default:
         enabled: yes
         target: 100%
+  ignore:
+    - "tests/*.py"
 
 comment: false

django_elastic_migrations/__init__.py

@@ -10,7 +10,7 @@
 from django_elastic_migrations.utils import loading
 from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger
 
-__version__ = '0.5.3'
+__version__ = '0.6.0'
 
 default_app_config = 'django_elastic_migrations.apps.DjangoElasticMigrationsConfig'  # pylint: disable=invalid-name
 
@@ -25,9 +25,14 @@
         'This should be the python path to the elasticsearch client '
         'to use for indexing.')
 
+logger.debug("using DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = {}".format(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT))
 
-es_client = loading.import_module_element(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT)
-
+try:
+    es_client = loading.import_module_element(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT)
+except ImportError:
+    logger.error("DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT {} not found. Please check your python path and django settings ".format(
+        settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT))
+    raise
 
 codebase_id = getattr(settings, 'DJANGO_ELASTIC_MIGRATIONS_GET_CODEBASE_ID', "")
 
@@ -59,7 +64,7 @@
 es_test_prefix = "test_"
 
 
-if 'test' in sys.argv:
+if 'test' in sys.argv and not environment_prefix == es_test_prefix:
     environment_prefix = '{}{}'.format(es_test_prefix, environment_prefix)
 
 from django_elastic_migrations import apps, indexes

django_elastic_migrations/apps.py

@@ -1,13 +1,10 @@
 # -*- coding: utf-8 -*-
+from __future__ import (absolute_import, division, print_function, unicode_literals)
+
 """
 django_elastic_migrations Django application initialization.
 """
 
-from __future__ import print_function
-from __future__ import absolute_import, unicode_literals
-
-import logging
-
 from django.apps import AppConfig
 
 

django_elastic_migrations/exceptions.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+from __future__ import (absolute_import, division, print_function, unicode_literals)
 from django.core.management import CommandError
 
 

django_elastic_migrations/indexes.py

@@ -3,6 +3,7 @@
 
 import sys
 
+import django
 from django.db import ProgrammingError
 from elasticsearch import TransportError
 from elasticsearch.helpers import expand_action, bulk
@@ -104,7 +105,7 @@ def create_index_model(cls, base_name):
             try:
                 index_model = DEMIndexModel.objects.create(name=base_name)
                 cls.index_models[base_name] = index_model
-            except ProgrammingError:
+            except (ProgrammingError, django.db.utils.OperationalError):
                 # the app is starting up and the database isn't available
                 pass
 
@@ -251,7 +252,7 @@ def update_index_models(cls):
             from django_elastic_migrations.models import Index as DEMIndexModel
             try:
                 cls.index_models = {i.name: i for i in DEMIndexModel.objects.all()}
-            except ProgrammingError:
+            except (ProgrammingError, django.db.utils.OperationalError):
                 # the app is starting up and the database isn't available
                 pass
 
@@ -727,7 +728,7 @@ def __init__(self, name, using=es_client, version_id=None):
         :param using: the elasticsearch client to use
         """
         prefixed_name = "{}{}".format(environment_prefix, name)
-        super(DEMIndex, self).__init__(prefixed_name, using)
+        super(DEMIndex, self).__init__(prefixed_name, using=using)
         self.__prefixed_name = prefixed_name
         self.__base_name = name
         self.__doc_type = None
@@ -892,9 +893,11 @@ def hash_matches(self, their_index_hash):
         our_index_hash, _ = self.get_index_hash_and_json()
         return our_index_hash == their_index_hash
 
-    def save(self):
+    def save(self, using=None):
+        if using is None:
+            using = es_client
         try:
-            super(DEMIndex, self).save()
+            super(DEMIndex, self).save(using=using)
         except ValueError as ex:
             if "Empty value" in ex.message and not self.get_active_version_index_name():
                 msg = (

django_elastic_migrations/management/commands/es.py

@@ -1,5 +1,4 @@
-
-from __future__ import print_function
+from __future__ import (absolute_import, division, print_function, unicode_literals)
 from django.core.management import BaseCommand, call_command, CommandError
 
 from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger

django_elastic_migrations/management/commands/es_activate.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+from __future__ import (absolute_import, division, print_function, unicode_literals)
 from django_elastic_migrations import DEMIndexManager
 from django_elastic_migrations.management.commands.es import ESCommand
 

django_elastic_migrations/management/commands/es_clear.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+from __future__ import (absolute_import, division, print_function, unicode_literals)
 from django_elastic_migrations import DEMIndexManager
 from django_elastic_migrations.management.commands.es import ESCommand
 from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger

django_elastic_migrations/management/commands/es_create.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+from __future__ import (absolute_import, division, print_function, unicode_literals)
 
 from django_elastic_migrations import DEMIndexManager
 from django_elastic_migrations.management.commands.es import ESCommand

django_elastic_migrations/management/commands/es_dangerous_reset.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+from __future__ import (absolute_import, division, print_function, unicode_literals)
 from django.core.management import call_command
 
 from django_elastic_migrations import DEMIndexManager

django_elastic_migrations/management/commands/es_deactivate.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+from __future__ import (absolute_import, division, print_function, unicode_literals)
 from django_elastic_migrations import DEMIndexManager
 from django_elastic_migrations.management.commands.es import ESCommand
 

django_elastic_migrations/management/commands/es_drop.py

@@ -1,13 +1,10 @@
-from __future__ import print_function
-import logging
-from django.core.management import CommandError
+from __future__ import (absolute_import, division, print_function, unicode_literals)
 
 from django_elastic_migrations import DEMIndexManager
 from django_elastic_migrations.exceptions import CannotDropAllIndexesWithoutForceArg
 from django_elastic_migrations.management.commands.es import ESCommand
 from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger
 
-
 logger = get_logger()
 
 

django_elastic_migrations/management/commands/es_list.py

@@ -1,4 +1,4 @@
-from __future__ import print_function
+from __future__ import (absolute_import, division, print_function, unicode_literals)
 from texttable import Texttable
 
 from django_elastic_migrations import DEMIndexManager
@@ -7,7 +7,7 @@
 from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger
 
 
-logger = get_logger()
+log = get_logger()
 
 
 class Command(ESCommand):
@@ -21,7 +21,7 @@ def add_arguments(self, parser):
         )
 
     def handle(self, *args, **options):
-        logger.info("Available Index Definitions:")
+        log.info("Available Index Definitions:")
 
         indexes, _, apply_all, _, _ = self.get_index_specifying_options(
             options, require_one_include_list=['es_only'])
@@ -80,8 +80,8 @@ def handle(self, *args, **options):
             except AttributeError:
                 raise FirstMigrationNotRunError()
 
-        logger.info(table.draw())
-        logger.info(
+        log.info(table.draw())
+        log.info(
             "An index version name is: \n"
             "{environment prefix}{index name}-{version primary key id}. \n" 
             "Most Django Elastic Migrations management commands take the \n"