diff --git a/.coveragerc b/.coveragerc index ff01f55..fb91fb2 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,7 +4,7 @@ data_file = .coverage source=django_elastic_migrations omit = test_settings - *migrations* + */migrations/* *admin.py *static* *templates* diff --git a/.gitignore b/.gitignore index c776a2d..899966e 100644 --- a/.gitignore +++ b/.gitignore @@ -61,8 +61,5 @@ output/*/index.html requirements/private.in requirements/private.txt -# tox environment temporary artifacts -tests/__init__.py - # Development task artifacts default.db diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..c0cf0b5 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,42 @@ +# Config file for automatic testing at travis-ci.org + +language: python +python: 3.6 + +dist: trusty + +matrix: + include: + - env: TOX_ENV=py27-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + python: 2.7 + - env: TOX_ENV=py27-django19-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + python: 2.7 +# TBD support - will be implemented in #5 +# allow_failures: +# - env: TOX_ENV=py27-django19-es62 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt +# python: 2.7 +# - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt +# python: 3.6 +# - env: TOX_ENV=py36-django18-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt +# python: 3.6 + +before_install: + - pip install --upgrade pip + # work around https://github.com/travis-ci/travis-ci/issues/8363 + - pip install codecov + - wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + - echo "deb $ES_APT_URL stable main" | sudo tee -a /etc/apt/sources.list.d/elk.list + - sudo apt-get update && sudo apt-get install elasticsearch -y + - sudo service elasticsearch start + +install: pip install -r requirements/travis.txt + +# sleep for elasticsearch +before_script: + - sleep 10 + +# command to run tests +script: tox -e $TOX_ENV + +after_success: + - codecov -e $TOX_ENV \ No newline at end of file diff --git a/CHANGELOG.rst b/CHANGELOG.rst new file mode 100644 index 0000000..f1f6e4d --- /dev/null +++ b/CHANGELOG.rst @@ -0,0 +1,11 @@ +Changelog +--------- + +0.6.0 (2018-08-01) +~~~~~~~~~~~~~~~~~~ +* Added test structure for py2 - GH #2 +* Renamed default log handler from ``django-elastic-migrations`` to ``django_elastic_migrations`` + +0.5.3 (2018-07-23) +~~~~~~~~~~~~~~~~~~ +* First basic release \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in index 1450764..dc582f6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,5 @@ include AUTHORS include CHANGELOG.rst include CONTRIBUTING.rst include LICENSE.txt -include README.rst +include README.md recursive-include django_elastic_migrations *.html *.png *.gif *js *.css *jpg *jpeg *svg *py diff --git a/Makefile b/Makefile index 48e10ad..99c3adc 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: clean coverage help \ - quality requirements selfcheck test test-all upgrade validate + quality requirements selfcheck syncdb load test test-all upgrade validate pylintrc .DEFAULT_GOAL := help @@ -29,17 +29,14 @@ clean: ## remove generated byte code, coverage reports, and build artifacts rm -fr dist/ rm -fr *.egg-info -coverage: clean ## generate and view HTML coverage report - py.test --cov-report html - $(BROWSER) htmlcov/index.html - upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in pip install -q pip-tools pip-compile --upgrade -o requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in pip-compile --upgrade -o requirements/quality.txt requirements/quality.in pip-compile --upgrade -o requirements/test.txt requirements/base.in requirements/test.in - # Let tox control the Django version for tests - sed '/django==/d' requirements/test.txt > requirements/test.tmp + pip-compile --upgrade -o requirements/travis.txt requirements/travis.in + # Let tox control the Django and elasticsearch-dsl version for tests + sed '/django==/d' requirements/test.txt | sed '/elasticsearch-dsl==/d' | sed '/elasticsearch==/d' > requirements/test.tmp mv requirements/test.tmp requirements/test.txt quality: ## check coding style with pycodestyle and pylint @@ -49,17 +46,31 @@ requirements: ## install development environment requirements pip install -qr requirements/dev.txt --exists-action w pip-sync requirements/dev.txt requirements/test.txt -test: clean ## run tests in the current virtualenv - py.test +coverage: clean ## check code coverage quickly with the default Python + coverage run ./manage.py test + coverage report -m + coverage html + $(BROWSER) htmlcov/index.html + +syncdb: clean ## setup local sqlite db + ./manage.py migrate --run-syncdb + +load: syncdb ## load fixtures into sqlite + ./manage.py loaddata tests/tests_initial.json + +test: syncdb ## run tests in the current virtualenv + ./manage.py test diff_cover: test diff-cover coverage.xml test-all: ## run tests on every supported Python/Django combination - tox -e quality tox validate: quality test ## run tests and quality checks selfcheck: ## check that the Makefile is well-formed @echo "The Makefile is well-formed." + +pylintrc: ## check that the Makefile is well-formed + edx_lint write pylintrc diff --git a/README.md b/README.md index 5b084c6..6c71134 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # Django Elastic Migrations Django Elastic Migrations provides a way to control the deployment of -multiple Elasticsearch schemas over time. +multiple Elasticsearch schemas over time. + +[![Build Status](https://travis-ci.com/HBS-HBX/django-elastic-migrations.svg?branch=master)](https://travis-ci.com/HBS-HBX/django-elastic-migrations) ## Overview @@ -275,13 +277,37 @@ and return the kwargs you would like to customize. This project uses `make` to manage the build process. Type `make help` to see the available `make` targets. +### Elasticsearch Docker Compose + +`docker-compose -f local.yml up` + +[See docs/docker_setup for more info](./docs/docker_setup.rst) + + ### Requirements +*`make upgrade`* upgrades the dependencies of the requirements to latest +version. This process also excludes `django` and `elasticsearch-dsl` +from the `requirements/test.txt` so they can be injected with different +versions by tox during matrix testing. -Then, `make requirements` runs the pip install. +*`make requirements`* runs the pip install. This project also uses [`pip-tools`](https://github.com/jazzband/pip-tools). The `requirements.txt` files are generated and pinned to latest versions -with `make upgrade`. +with `make upgrade`. + +### Running Tests Locally + +Run `make test`. To run all tests and quality checks locally, +run `make test-all`. + +To just run linting, `make quality`. Please note that if any of the +linters return a nonzero code, it will give an `InvocationError` error +at the end. See [tox's documentation for `InvocationError`](https://tox.readthedocs.io/en/latest/example/general.html#understanding-invocationerror-exit-codes) +for more information. + +We use `edx_lint` to compile `pylintrc`. To update the rules, +change `pylintrc_tweaks` and run `make pylintrc`. ### Updating Egg Info diff --git a/codecov.yml b/codecov.yml index 4da4768..be7cafb 100644 --- a/codecov.yml +++ b/codecov.yml @@ -8,5 +8,7 @@ coverage: default: enabled: yes target: 100% + ignore: + - "tests/*.py" comment: false diff --git a/django_elastic_migrations/__init__.py b/django_elastic_migrations/__init__.py index 7bda52c..5a33ff6 100644 --- a/django_elastic_migrations/__init__.py +++ b/django_elastic_migrations/__init__.py @@ -10,7 +10,7 @@ from django_elastic_migrations.utils import loading from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger -__version__ = '0.5.3' +__version__ = '0.6.0' default_app_config = 'django_elastic_migrations.apps.DjangoElasticMigrationsConfig' # pylint: disable=invalid-name @@ -25,9 +25,14 @@ 'This should be the python path to the elasticsearch client ' 'to use for indexing.') +logger.debug("using DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = {}".format(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT)) -es_client = loading.import_module_element(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT) - +try: + es_client = loading.import_module_element(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT) +except ImportError: + logger.error("DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT {} not found. Please check your python path and django settings ".format( + settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT)) + raise codebase_id = getattr(settings, 'DJANGO_ELASTIC_MIGRATIONS_GET_CODEBASE_ID', "") @@ -59,7 +64,7 @@ es_test_prefix = "test_" -if 'test' in sys.argv: +if 'test' in sys.argv and not environment_prefix == es_test_prefix: environment_prefix = '{}{}'.format(es_test_prefix, environment_prefix) from django_elastic_migrations import apps, indexes diff --git a/django_elastic_migrations/apps.py b/django_elastic_migrations/apps.py index 7078cc0..7255a6c 100644 --- a/django_elastic_migrations/apps.py +++ b/django_elastic_migrations/apps.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- +from __future__ import (absolute_import, division, print_function, unicode_literals) + """ django_elastic_migrations Django application initialization. """ -from __future__ import print_function -from __future__ import absolute_import, unicode_literals - -import logging - from django.apps import AppConfig diff --git a/django_elastic_migrations/exceptions.py b/django_elastic_migrations/exceptions.py index e2cbdfc..4260fe4 100644 --- a/django_elastic_migrations/exceptions.py +++ b/django_elastic_migrations/exceptions.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django.core.management import CommandError diff --git a/django_elastic_migrations/indexes.py b/django_elastic_migrations/indexes.py index 8fdd32c..1f98ac3 100644 --- a/django_elastic_migrations/indexes.py +++ b/django_elastic_migrations/indexes.py @@ -3,6 +3,7 @@ import sys +import django from django.db import ProgrammingError from elasticsearch import TransportError from elasticsearch.helpers import expand_action, bulk @@ -104,7 +105,7 @@ def create_index_model(cls, base_name): try: index_model = DEMIndexModel.objects.create(name=base_name) cls.index_models[base_name] = index_model - except ProgrammingError: + except (ProgrammingError, django.db.utils.OperationalError): # the app is starting up and the database isn't available pass @@ -251,7 +252,7 @@ def update_index_models(cls): from django_elastic_migrations.models import Index as DEMIndexModel try: cls.index_models = {i.name: i for i in DEMIndexModel.objects.all()} - except ProgrammingError: + except (ProgrammingError, django.db.utils.OperationalError): # the app is starting up and the database isn't available pass @@ -727,7 +728,7 @@ def __init__(self, name, using=es_client, version_id=None): :param using: the elasticsearch client to use """ prefixed_name = "{}{}".format(environment_prefix, name) - super(DEMIndex, self).__init__(prefixed_name, using) + super(DEMIndex, self).__init__(prefixed_name, using=using) self.__prefixed_name = prefixed_name self.__base_name = name self.__doc_type = None @@ -892,9 +893,11 @@ def hash_matches(self, their_index_hash): our_index_hash, _ = self.get_index_hash_and_json() return our_index_hash == their_index_hash - def save(self): + def save(self, using=None): + if using is None: + using = es_client try: - super(DEMIndex, self).save() + super(DEMIndex, self).save(using=using) except ValueError as ex: if "Empty value" in ex.message and not self.get_active_version_index_name(): msg = ( diff --git a/django_elastic_migrations/management/commands/es.py b/django_elastic_migrations/management/commands/es.py index d8b0434..ac0a807 100644 --- a/django_elastic_migrations/management/commands/es.py +++ b/django_elastic_migrations/management/commands/es.py @@ -1,5 +1,4 @@ - -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django.core.management import BaseCommand, call_command, CommandError from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger diff --git a/django_elastic_migrations/management/commands/es_activate.py b/django_elastic_migrations/management/commands/es_activate.py index dee34eb..cb33db9 100644 --- a/django_elastic_migrations/management/commands/es_activate.py +++ b/django_elastic_migrations/management/commands/es_activate.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations import DEMIndexManager from django_elastic_migrations.management.commands.es import ESCommand diff --git a/django_elastic_migrations/management/commands/es_clear.py b/django_elastic_migrations/management/commands/es_clear.py index 6984169..6ea29f8 100644 --- a/django_elastic_migrations/management/commands/es_clear.py +++ b/django_elastic_migrations/management/commands/es_clear.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations import DEMIndexManager from django_elastic_migrations.management.commands.es import ESCommand from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger diff --git a/django_elastic_migrations/management/commands/es_create.py b/django_elastic_migrations/management/commands/es_create.py index 7a620ab..9a12109 100644 --- a/django_elastic_migrations/management/commands/es_create.py +++ b/django_elastic_migrations/management/commands/es_create.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations import DEMIndexManager from django_elastic_migrations.management.commands.es import ESCommand diff --git a/django_elastic_migrations/management/commands/es_dangerous_reset.py b/django_elastic_migrations/management/commands/es_dangerous_reset.py index 8469f96..5c64cd2 100644 --- a/django_elastic_migrations/management/commands/es_dangerous_reset.py +++ b/django_elastic_migrations/management/commands/es_dangerous_reset.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django.core.management import call_command from django_elastic_migrations import DEMIndexManager diff --git a/django_elastic_migrations/management/commands/es_deactivate.py b/django_elastic_migrations/management/commands/es_deactivate.py index 70bbab8..d48b52c 100644 --- a/django_elastic_migrations/management/commands/es_deactivate.py +++ b/django_elastic_migrations/management/commands/es_deactivate.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations import DEMIndexManager from django_elastic_migrations.management.commands.es import ESCommand diff --git a/django_elastic_migrations/management/commands/es_drop.py b/django_elastic_migrations/management/commands/es_drop.py index 90a50c3..f762743 100644 --- a/django_elastic_migrations/management/commands/es_drop.py +++ b/django_elastic_migrations/management/commands/es_drop.py @@ -1,13 +1,10 @@ -from __future__ import print_function -import logging -from django.core.management import CommandError +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations import DEMIndexManager from django_elastic_migrations.exceptions import CannotDropAllIndexesWithoutForceArg from django_elastic_migrations.management.commands.es import ESCommand from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger - logger = get_logger() diff --git a/django_elastic_migrations/management/commands/es_list.py b/django_elastic_migrations/management/commands/es_list.py index 071b897..073a866 100644 --- a/django_elastic_migrations/management/commands/es_list.py +++ b/django_elastic_migrations/management/commands/es_list.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from texttable import Texttable from django_elastic_migrations import DEMIndexManager @@ -7,7 +7,7 @@ from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger -logger = get_logger() +log = get_logger() class Command(ESCommand): @@ -21,7 +21,7 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - logger.info("Available Index Definitions:") + log.info("Available Index Definitions:") indexes, _, apply_all, _, _ = self.get_index_specifying_options( options, require_one_include_list=['es_only']) @@ -80,8 +80,8 @@ def handle(self, *args, **options): except AttributeError: raise FirstMigrationNotRunError() - logger.info(table.draw()) - logger.info( + log.info(table.draw()) + log.info( "An index version name is: \n" "{environment prefix}{index name}-{version primary key id}. \n" "Most Django Elastic Migrations management commands take the \n" diff --git a/django_elastic_migrations/management/commands/es_update.py b/django_elastic_migrations/management/commands/es_update.py index d762e4c..69b6985 100644 --- a/django_elastic_migrations/management/commands/es_update.py +++ b/django_elastic_migrations/management/commands/es_update.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from dateutil.parser import parse as dateutil_parse @@ -7,7 +7,7 @@ from django_elastic_migrations.utils.multiprocessing_utils import USE_ALL_WORKERS -logger = get_logger() +log = get_logger() class Command(ESCommand): @@ -60,7 +60,7 @@ def handle(self, *args, **options): if start_date is not None: if resume_mode: - logger.warning("--start takes precedence over --resume mode!") + log.warning("--start takes precedence over --resume mode!") start_date = dateutil_parse(start_date) if start_date: resume_mode = False diff --git a/django_elastic_migrations/models.py b/django_elastic_migrations/models.py index 305a66d..4eb39fd 100644 --- a/django_elastic_migrations/models.py +++ b/django_elastic_migrations/models.py @@ -1030,7 +1030,7 @@ def set_task_kwargs(self, kwargs): new_kwargs = {} for required_attribute in self.REQUIRED_TASK_KWARGS: new_kwargs[required_attribute] = kwargs[required_attribute] - self.task_kwargs = json.dumps(new_kwargs) + self.task_kwargs = json.dumps(new_kwargs, sort_keys=True) def perform_action(self, dem_index, *args, **kwargs): diff --git a/django_elastic_migrations/utils/__init__.py b/django_elastic_migrations/utils/__init__.py index 0170cf4..77b4367 100644 --- a/django_elastic_migrations/utils/__init__.py +++ b/django_elastic_migrations/utils/__init__.py @@ -1,2 +1,2 @@ -from __future__ import print_function -import importlib \ No newline at end of file +from __future__ import (absolute_import, division, print_function, unicode_literals) +import importlib diff --git a/django_elastic_migrations/utils/django_elastic_migrations_log.py b/django_elastic_migrations/utils/django_elastic_migrations_log.py index 3e1e569..c23b9e6 100644 --- a/django_elastic_migrations/utils/django_elastic_migrations_log.py +++ b/django_elastic_migrations/utils/django_elastic_migrations_log.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) import logging from multiprocessing_logging import install_mp_handler @@ -6,7 +6,7 @@ install_mp_handler() -def get_logger(name="django-elastic-migrations"): +def get_logger(name="django_elastic_migrations"): real_logger = logging.getLogger(name) for level in ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG']: setattr(real_logger, level, getattr(logging, level)) diff --git a/django_elastic_migrations/utils/es_utils.py b/django_elastic_migrations/utils/es_utils.py index d71b10d..334ee28 100644 --- a/django_elastic_migrations/utils/es_utils.py +++ b/django_elastic_migrations/utils/es_utils.py @@ -5,7 +5,7 @@ def get_index_hash_and_json(index): spec = index.to_dict() - json_str = json.dumps(spec) - hash = hashlib.md5() - hash.update(json_str) - return hash.hexdigest(), json_str + json_str = json.dumps(spec, sort_keys=True) + md5_hash = hashlib.md5() + md5_hash.update(json_str) + return md5_hash.hexdigest(), json_str diff --git a/django_elastic_migrations/utils/loading.py b/django_elastic_migrations/utils/loading.py index e069587..b1b92d8 100644 --- a/django_elastic_migrations/utils/loading.py +++ b/django_elastic_migrations/utils/loading.py @@ -1,7 +1,5 @@ # encoding: utf-8 - -from __future__ import print_function -from __future__ import absolute_import, division, print_function, unicode_literals +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations.utils import importlib diff --git a/django_elastic_migrations/utils/test_utils.py b/django_elastic_migrations/utils/test_utils.py new file mode 100644 index 0000000..3e0bdbb --- /dev/null +++ b/django_elastic_migrations/utils/test_utils.py @@ -0,0 +1,15 @@ +from __future__ import (absolute_import, division, print_function, unicode_literals) +from django.test import TestCase + +from django_elastic_migrations import DEMIndexManager + + +class DEMTestCase(TestCase): + + def setUp(self): + DEMIndexManager.test_pre_setup() + super(DEMTestCase, self)._pre_setup() + + def tearDown(self): + DEMIndexManager.test_post_teardown() + super(DEMTestCase, self).tearDown() diff --git a/docs/docker_setup.rst b/docs/docker_setup.rst new file mode 100644 index 0000000..42810ce --- /dev/null +++ b/docs/docker_setup.rst @@ -0,0 +1,59 @@ +Getting Elasticsearch Up and Running Locally With Docker +======================================================== + +.. index:: Docker + +The steps below will get you up and running elasticsearch in a local development environment. +All of these commands assume you are in the project root. + + +Prerequisites +------------- + +* Docker; if you don't have it yet, follow the `installation instructions`_; +* Docker Compose; refer to the official documentation for the `installation guide`_. + +.. _`installation instructions`: https://docs.docker.com/install/#supported-platforms +.. _`installation guide`: https://docs.docker.com/compose/install/ + + +Build the Stack (currently unnecessary) +--------------------------------------- + +Currently, the dev stack consists of prebuilt images. In the future, +if custom docker images are added to the compose file, this may be necessary:: + + $ docker-compose -f local.yml build + + +Run the Stack +------------- + +This brings up Elasticsearch. The first time it is run it might take a while to get started, but subsequent runs will occur quickly. + +Open a terminal at the project root and run the following for local development:: + + $ docker-compose -f local.yml up + +You can also set the environment variable ``COMPOSE_FILE`` pointing to ``local.yml`` like this:: + + $ export COMPOSE_FILE=local.yml + +And then run:: + + $ docker-compose up + +To run in a detached (background) mode, just:: + + $ docker-compose up -d + + +Tips & Tricks +------------- + +Activate a Docker Machine +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This tells our computer that all future commands are specifically for the dev1 machine. Using the ``eval`` command we can switch machines as needed.:: + + $ eval "$(docker-machine env dev1)" diff --git a/local.yml b/local.yml new file mode 100644 index 0000000..c80ff43 --- /dev/null +++ b/local.yml @@ -0,0 +1,43 @@ +# creates a dev elasticsearch (& Kibana, if you want) +# start: +# docker-compose up +# stop: +# docker-compose down +# delete: +# docker-compose kill + +version: '2.1' + +services: + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:6.0.1 + ports: + - 9200:9200 + - 9300:9300 + environment: + - "discovery.type=single-node" + - "transport.host=127.0.0.1" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200/_count"] + interval: 10s + timeout: 10s + retries: 10 + +# uncomment to get kibana, which can be useful for analyzing index contents +# this is currently not a required part of testing infrastructure, just left here for convenience +# kibana: +# image: docker.elastic.co/kibana/kibana:6.0.1 +# depends_on: +# elastic: +# condition: service_healthy +# links: +# - elastic +# ports: +# - 5601:5601 +# environment: +# - "ELASTICSEARCH_URL=http://elastic:9200" +# healthcheck: +# test: ["CMD", "curl", "-f", "http://localhost:5601"] +# interval: 10s +# timeout: 10s +# retries: 10 \ No newline at end of file diff --git a/pylintrc b/pylintrc index a21bfca..ab70f89 100644 --- a/pylintrc +++ b/pylintrc @@ -1,32 +1,274 @@ +# *************************** +# ** DO NOT EDIT THIS FILE ** +# *************************** +# +# This file was generated by edx-lint: http://github.com/edx/edx-lint +# +# If you want to change this file, you have two choices, depending on whether +# you want to make a local change that applies only to this repo, or whether +# you want to make a central change that applies to all repos using edx-lint. +# +# LOCAL CHANGE: +# +# 1. Edit the local pylintrc_tweaks file to add changes just to this +# repo's file. +# +# 2. Run: +# +# $ edx_lint write pylintrc +# +# 3. This will modify the local file. Submit a pull request to get it +# checked in so that others will benefit. +# +# +# CENTRAL CHANGE: +# +# 1. Edit the pylintrc file in the edx-lint repo at +# https://github.com/edx/edx-lint/blob/master/edx_lint/files/pylintrc +# +# 2. Make a new version of edx_lint, which involves the usual steps of +# incrementing the version number, submitting and reviewing a pull +# request, and updating the edx-lint version reference in this repo. +# +# 3. Install the newer version of edx-lint. +# +# 4. Run: +# +# $ edx_lint write pylintrc +# +# 5. This will modify the local file. Submit a pull request to get it +# checked in so that others will benefit. +# +# +# +# +# +# STAY AWAY FROM THIS FILE! +# +# +# +# +# +# SERIOUSLY. +# +# ------------------------------ [MASTER] ignore = migrations persistent = yes -load-plugins = caniusepython3.pylint_checker,pylint_django,pylint_celery +load-plugins = caniusepython3.pylint_checker,edx_lint.pylint,pylint_django [MESSAGES CONTROL] -disable = - locally-disabled, - locally-enabled, - too-few-public-methods, - bad-builtin, - star-args, - abstract-class-not-used, - abstract-class-little-used, - no-init, - fixme, - logging-format-interpolation, - too-many-lines, - no-self-use, - too-many-ancestors, - too-many-instance-attributes, - too-few-public-methods, - too-many-public-methods, - too-many-return-statements, - too-many-branches, - too-many-arguments, - too-many-locals, - unused-wildcard-import, - duplicate-code +enable = + # These are controlled by explicit choices in the pylintrc files + blacklisted-name, + line-too-long, + # These affect the correctness of the code + syntax-error, + init-is-generator, + return-in-init, + function-redefined, + not-in-loop, + return-outside-function, + yield-outside-function, + return-arg-in-generator, + nonexistent-operator, + duplicate-argument-name, + abstract-class-instantiated, + bad-reversed-sequence, + continue-in-finally, + method-hidden, + access-member-before-definition, + no-method-argument, + no-self-argument, + invalid-slots-object, + assigning-non-slot, + invalid-slots, + inherit-non-class, + inconsistent-mro, + duplicate-bases, + non-iterator-returned, + unexpected-special-method-signature, + invalid-length-returned, + import-error, + used-before-assignment, + undefined-variable, + undefined-all-variable, + invalid-all-object, + no-name-in-module, + unbalance-tuple-unpacking, + unpacking-non-sequence, + bad-except-order, + raising-bad-type, + misplaced-bare-raise, + raising-non-exception, + nonimplemented-raised, + catching-non-exception, + slots-on-old-class, + super-on-old-class, + bad-super-call, + missing-super-argument, + no-member, + not-callable, + assignment-from-no-return, + no-value-for-parameter, + too-many-function-args, + unexpected-keyword-arg, + redundant-keyword-arg, + invalid-sequence-index, + invalid-slice-index, + assignment-from-none, + not-context-manager, + invalid-unary-operand-type, + unsupported-binary-operation, + repeated-keyword, + not-an-iterable, + not-a-mapping, + unsupported-membership-test, + unsubscriptable-object, + logging-unsupported-format, + logging-too-many-args, + logging-too-few-args, + bad-format-character, + truncated-format-string, + mixed-fomat-string, + format-needs-mapping, + missing-format-string-key, + too-many-format-args, + too-few-format-args, + bad-str-strip-call, + model-unicode-not-callable, + super-method-not-called, + non-parent-method-called, + test-inherits-tests, + translation-of-non-string, + redefined-variable-type, + cyclical-import, + unreachable, + dangerous-default-value, + pointless-statement, + pointless-string-statement, + expression-not-assigned, + duplicate-key, + confusing-with-statement, + using-constant-test, + lost-exception, + assert-on-tuple, + attribute-defined-outside-init, + bad-staticmethod-argument, + arguments-differ, + signature-differs, + abstract-method, + super-init-not-called, + relative-import, + import-self, + misplaced-future, + invalid-encoded-data, + global-variable-undefined, + redefined-outer-name, + redefined-builtin, + redefined-in-handler, + undefined-loop-variable, + cell-var-from-loop, + duplicate-except, + nonstandard-exception, + binary-op-exception, + property-on-old-class, + bad-format-string-key, + unused-format-string-key, + bad-format-string, + missing-format-argument-key, + unused-format-string-argument, + format-combined-specification, + missing-format-attribute, + invalid-format-index, + anomalous-backslash-in-string, + anomalous-unicode-escape-in-string, + bad-open-mode, + boolean-datetime, + # Checking failed for some reason + fatal, + astroid-error, + parse-error, + method-check-failed, + django-not-available, + raw-checker-failed, + django-not-available-placeholder, + # Documentation is important + empty-docstring, + invalid-characters-in-docstring, + missing-docstring, + wrong-spelling-in-comment, + wrong-spelling-in-docstring, + # Unused code should be deleted + unused-import, + unused-variable, + unused-argument, + # These are dangerous! + exec-used, + eval-used, + # These represent idiomatic python. Not adhering to them + # will raise red flags with future readers. + bad-classmethod-argument, + bad-mcs-classmethod-argument, + bad-mcs-method-argument, + bad-whitespace, + consider-iterating-dictionary, + consider-using-enumerate, + literal-used-as-attribute, + multiple-imports, + multiple-statements, + old-style-class, + simplifiable-range, + singleton-comparison, + superfluous-parens, + unidiomatic-typecheck, + unneeded-not, + wrong-assert-type, + simplifiable-if-statement, + no-classmethod-decorator, + no-staticmethod-decorator, + unnecessary-pass, + unnecessary-lambda, + useless-else-on-loop, + unnecessary-semicolon, + reimported, + global-variable-not-assigned, + global-at-module-level, + bare-except, + broad-except, + logging-not-lazy, + redundant-unittest-assert, + model-missing-unicode, + model-has-unicode, + model-no-explicit-unicode, + protected-access, + # Don't use things that are deprecated + deprecated-module, + deprecated-method, + # These help manage code complexity + too-many-nested-blocks, + too-many-statements, + too-many-boolean-expressions, + # Consistent import order makes finding where code is + # imported from easier + ungrouped-imports, + wrong-import-order, + wrong-import-position, + wildcard-import, + # These should be auto-fixed by any competent editor + missing-final-newline, + mixed-line-endings, + trailing-newlines, + trailing-whitespace, + unexpected-line-ending-format, + mixed-indentation, + # These attempt to limit pylint line-noise + bad-option-value, + unrecognized-inline-option, + useless-suppression, + bad-inline-option, + deprecated-pragma, +disable = C0111,unused-argument [REPORTS] output-format = text @@ -49,7 +291,7 @@ inlinevar-rgx = [A-Za-z_][A-Za-z0-9_]*$ good-names = f,i,j,k,db,ex,Run,_,__ bad-names = foo,bar,baz,toto,tutu,tata no-docstring-rgx = __.*__$|test_.+|setUp$|setUpClass$|tearDown$|tearDownClass$|Meta$ -docstring-min-length = -1 +docstring-min-length = 5 [FORMAT] max-line-length = 120 @@ -72,7 +314,7 @@ ignore-imports = no ignore-mixin-members = yes ignored-classes = SQLObject unsafe-load-any-extension = yes -generated-members = +generated-members = REQUEST, acl_users, aq_parent, @@ -98,7 +340,7 @@ generated-members = [VARIABLES] init-import = no dummy-variables-rgx = _|dummy|unused|.*_unused -additional-builtins = +additional-builtins = [CLASSES] defining-attr-methods = __init__,__new__,setUp @@ -119,11 +361,11 @@ max-public-methods = 20 [IMPORTS] deprecated-modules = regsub,TERMIOS,Bastion,rexec -import-graph = -ext-import-graph = -int-import-graph = +import-graph = +ext-import-graph = +int-import-graph = [EXCEPTIONS] overgeneral-exceptions = Exception -# 1a67033d4799199101eddf63b8ed0bef3e61bda7 +# 701d58a579f575603a2045f5e061520c889b54cf diff --git a/pylintrc_tweaks b/pylintrc_tweaks new file mode 100644 index 0000000..848752a --- /dev/null +++ b/pylintrc_tweaks @@ -0,0 +1,7 @@ +# pylintrc tweaks - put changes here, run `edx_lint write pylintrc` and it will generate pylintrc +[MASTER] +ignore = migrations +load-plugins = caniusepython3.pylint_checker,edx_lint.pylint,pylint_django + +[MESSAGES CONTROL] +disable=C0111,unused-argument diff --git a/requirements/base.in b/requirements/base.in index bec2246..5e661e0 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,5 +1,6 @@ # Core requirements for using this application -Django==1.9.8 # Web application framework -elasticsearch-dsl # Used for connecting to elasticsearch +Django # Web application framework +elasticsearch-dsl==6.1.0 # For interacting with Elaticsearch texttable # Printing tabular data from management commands +multiprocessing-logging # for logging in ./manage.py es_update --workers \ No newline at end of file diff --git a/requirements/dev.in b/requirements/dev.in index af2d09f..ea485be 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -1,7 +1,8 @@ # Additional requirements for development of this application +edx-lint # includes pylint, pylint-django, etc; configured use pylintrc diff-cover # Changeset diff test coverage pip-tools # Requirements file management tox # virtualenv management for tests tox-battery # Makes tox aware of requirements file changes -wheel # For generation of wheels for PyPI +wheel # For generation of wheels for PyPI \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index 5d72604..1697820 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,41 +5,55 @@ # pip-compile --output-file requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in # argparse==1.4.0 # via caniusepython3 -backports.functools-lru-cache==1.5 # via caniusepython3 -caniusepython3==6.0.0 +astroid==1.5.2 # via edx-lint, pylint, pylint-celery +backports.functools-lru-cache==1.5 # via astroid, caniusepython3, pylint +caniusepython3==7.0.0 certifi==2018.4.16 # via requests chardet==3.0.4 # via requests -click==6.7 # via pip-tools -configparser==3.5.0 # via pydocstyle -diff-cover==1.0.2 +click-log==0.1.8 # via edx-lint +click==6.7 # via click-log, edx-lint, pip-tools +configparser==3.5.0 # via pydocstyle, pylint +diff-cover==1.0.4 distlib==0.2.7 # via caniusepython3 -django==1.9.8 +django==1.11.15 +edx-lint==0.5.5 elasticsearch-dsl==6.1.0 -elasticsearch==6.2.0 # via elasticsearch-dsl +elasticsearch==6.3.0 # via elasticsearch-dsl +enum34==1.1.6 # via astroid first==2.0.1 # via pip-tools futures==3.2.0 # via caniusepython3, isort -idna==2.6 # via requests -inflect==0.2.5 # via jinja2-pluralize +idna==2.7 # via requests +inflect==1.0.0 # via jinja2-pluralize ipaddress==1.0.22 # via elasticsearch-dsl isort==4.3.4 jinja2-pluralize==0.3.0 # via diff-cover jinja2==2.10 # via diff-cover, jinja2-pluralize +lazy-object-proxy==1.3.1 # via astroid markupsafe==1.0 # via jinja2 -packaging==17.1 # via caniusepython3 +mccabe==0.6.1 # via pylint +multiprocessing-logging==0.2.6 +packaging==17.1 # via caniusepython3, tox pip-tools==2.0.2 -pluggy==0.6.0 # via tox -py==1.5.3 # via tox +pluggy==0.7.1 # via tox +py==1.5.4 # via tox pycodestyle==2.4.0 pydocstyle==2.1.1 pygments==2.2.0 # via diff-cover +pylint-celery==0.3 # via edx-lint +pylint-django==0.7.2 # via edx-lint +pylint-plugin-utils==0.4 # via pylint-celery, pylint-django +pylint==1.7.1 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils pyparsing==2.2.0 # via packaging -python-dateutil==2.7.2 # via elasticsearch-dsl -requests==2.18.4 # via caniusepython3 -six==1.11.0 # via diff-cover, elasticsearch-dsl, packaging, pip-tools, pydocstyle, python-dateutil, tox +python-dateutil==2.7.3 # via elasticsearch-dsl +pytz==2018.5 # via django +requests==2.19.1 # via caniusepython3 +singledispatch==3.4.0.3 # via astroid, pylint +six==1.11.0 # via astroid, diff-cover, edx-lint, elasticsearch-dsl, packaging, pip-tools, pydocstyle, pylint, python-dateutil, singledispatch, tox snowballstemmer==1.2.1 # via pydocstyle -texttable==1.2.1 +texttable==1.4.0 tox-battery==0.5.1 -tox==3.0.0 -urllib3==1.22 # via elasticsearch, requests -virtualenv==15.2.0 # via tox -wheel==0.31.0 +tox==3.1.2 +urllib3==1.23 # via elasticsearch, requests +virtualenv==16.0.0 # via tox +wheel==0.31.1 +wrapt==1.10.11 # via astroid diff --git a/requirements/quality.in b/requirements/quality.in index 6e6fdc8..803c7e3 100644 --- a/requirements/quality.in +++ b/requirements/quality.in @@ -1,6 +1,7 @@ # Requirements for code quality checks caniusepython3 # Additional Python 3 compatibility pylint checks +edx-lint # includes pylint, pylint-django, etc; configured use pylintrc isort # to standardize order of imports pycodestyle # PEP 8 compliance validation pydocstyle # PEP 257 compliance validation diff --git a/requirements/quality.txt b/requirements/quality.txt index 4c441c5..6aa44e2 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -5,20 +5,33 @@ # pip-compile --output-file requirements/quality.txt requirements/quality.in # argparse==1.4.0 # via caniusepython3 -backports.functools-lru-cache==1.5 # via caniusepython3 -caniusepython3==6.0.0 +astroid==1.5.2 # via edx-lint, pylint, pylint-celery +backports.functools-lru-cache==1.5 # via astroid, caniusepython3, pylint +caniusepython3==7.0.0 certifi==2018.4.16 # via requests chardet==3.0.4 # via requests -configparser==3.5.0 # via pydocstyle +click-log==0.1.8 # via edx-lint +click==6.7 # via click-log, edx-lint +configparser==3.5.0 # via pydocstyle, pylint distlib==0.2.7 # via caniusepython3 +edx-lint==0.5.5 +enum34==1.1.6 # via astroid futures==3.2.0 # via caniusepython3, isort -idna==2.6 # via requests +idna==2.7 # via requests isort==4.3.4 +lazy-object-proxy==1.3.1 # via astroid +mccabe==0.6.1 # via pylint packaging==17.1 # via caniusepython3 pycodestyle==2.4.0 pydocstyle==2.1.1 +pylint-celery==0.3 # via edx-lint +pylint-django==0.7.2 # via edx-lint +pylint-plugin-utils==0.4 # via pylint-celery, pylint-django +pylint==1.7.1 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils pyparsing==2.2.0 # via packaging -requests==2.18.4 # via caniusepython3 -six==1.11.0 # via packaging, pydocstyle +requests==2.19.1 # via caniusepython3 +singledispatch==3.4.0.3 # via astroid, pylint +six==1.11.0 # via astroid, edx-lint, packaging, pydocstyle, pylint, singledispatch snowballstemmer==1.2.1 # via pydocstyle -urllib3==1.22 # via requests +urllib3==1.23 # via requests +wrapt==1.10.11 # via astroid diff --git a/requirements/test.in b/requirements/test.in index 9758da5..a3e1454 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -1,4 +1,6 @@ # Requirements for test runs. -pytest-cov # pytest extension for code coverage statistics -pytest-django # pytest extension for better Django support +codecov # for https://codecov.io/ github integration +coverage # for quick, one-off coverage tests +tox # virtualenv management for tests +tox-battery # Makes tox aware of requirements file changes \ No newline at end of file diff --git a/requirements/test.txt b/requirements/test.txt index 27e76a9..a4f2b0c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,18 +4,23 @@ # # pip-compile --output-file requirements/test.txt requirements/base.in requirements/test.in # -attrs==17.4.0 # via pytest -coverage==4.5.1 # via pytest-cov -elasticsearch-dsl==6.1.0 -elasticsearch==6.2.0 # via elasticsearch-dsl -funcsigs==1.0.2 # via pytest +certifi==2018.4.16 # via requests +chardet==3.0.4 # via requests +codecov==2.0.15 +coverage==4.5.1 +idna==2.7 # via requests ipaddress==1.0.22 # via elasticsearch-dsl -more-itertools==4.1.0 # via pytest -pluggy==0.6.0 # via pytest -py==1.5.3 # via pytest -pytest-cov==2.5.1 -pytest==3.5.1 # via pytest-cov, pytest-django -python-dateutil==2.7.2 # via elasticsearch-dsl -six==1.11.0 # via elasticsearch-dsl, more-itertools, pytest, python-dateutil -texttable==1.2.1 -urllib3==1.22 # via elasticsearch +multiprocessing-logging==0.2.6 +packaging==17.1 # via tox +pluggy==0.7.1 # via tox +py==1.5.4 # via tox +pyparsing==2.2.0 # via packaging +python-dateutil==2.7.3 # via elasticsearch-dsl +pytz==2018.5 # via django +requests==2.19.1 # via codecov +six==1.11.0 # via elasticsearch-dsl, packaging, python-dateutil, tox +texttable==1.4.0 +tox-battery==0.5.1 +tox==3.1.2 +urllib3==1.23 # via elasticsearch, requests +virtualenv==16.0.0 # via tox diff --git a/requirements/travis.in b/requirements/travis.in new file mode 100644 index 0000000..f1f15dc --- /dev/null +++ b/requirements/travis.in @@ -0,0 +1,5 @@ +# Requirements for running tests in Travis + +codecov # Code coverage reporting +tox # Virtualenv management for tests +tox-battery # Makes tox aware of requirements file changes \ No newline at end of file diff --git a/requirements/travis.txt b/requirements/travis.txt new file mode 100644 index 0000000..750c5b7 --- /dev/null +++ b/requirements/travis.txt @@ -0,0 +1,21 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements/travis.txt requirements/travis.in +# +certifi==2018.4.16 # via requests +chardet==3.0.4 # via requests +codecov==2.0.15 +coverage==4.5.1 # via codecov +idna==2.7 # via requests +packaging==17.1 # via tox +pluggy==0.7.1 # via tox +py==1.5.4 # via tox +pyparsing==2.2.0 # via packaging +requests==2.19.1 # via codecov +six==1.11.0 # via packaging, tox +tox-battery==0.5.1 +tox==3.1.2 +urllib3==1.23 # via requests +virtualenv==16.0.0 # via tox diff --git a/setup.py b/setup.py index 7bc6f55..596c343 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # pylint: disable=C0111,W6005,W6100 -from __future__ import print_function from __future__ import absolute_import, print_function import os @@ -33,8 +32,8 @@ def get_version(*file_paths): os.system("git push origin %s" % VERSION) sys.exit() -README = open(os.path.join(os.path.dirname(__file__), 'README.md')).read() -CHANGELOG = open(os.path.join(os.path.dirname(__file__), 'CHANGELOG.md')).read() +README = open('README.md').read() +CHANGELOG = open('CHANGELOG.rst').read() setup( name='django-elastic-migrations', @@ -43,7 +42,7 @@ def get_version(*file_paths): long_description=README + '\n\n' + CHANGELOG, author='Harvard Business School, HBX Department', author_email='pnore@hbs.edu', - url='https://github.com/HBS-HBX/django_elastic_migrations', + url='https://github.com/HBS-HBX/django-elastic-migrations', packages=[ 'django_elastic_migrations', ], diff --git a/test_settings.py b/test_settings.py index 5c7f1ea..4cdb239 100644 --- a/test_settings.py +++ b/test_settings.py @@ -8,8 +8,17 @@ from __future__ import print_function from __future__ import absolute_import, unicode_literals +import logging +import os +import sys +from logging import config as logging_config +import subprocess + +import django from os.path import abspath, dirname, join +DEBUG = True + def root(*args): """ @@ -33,6 +42,7 @@ def root(*args): 'django.contrib.auth', 'django.contrib.contenttypes', 'django_elastic_migrations', + 'tests' ) ROOT_URLCONF = 'django_elastic_migrations.urls' @@ -41,11 +51,55 @@ def root(*args): # parameters to pass to base Elasticsearch() instance # https://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch -# override in app vars ELASTICSEARCH_PARAMS = { 'hosts': [ 'localhost:9200' ], } -DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = 'django_elastic_migrations.utils.es_utils.DEFAULT_ES_CLIENT' +ELASTICSEARCH_INDEX_SETTINGS = { + "number_of_shards": 1, + "number_of_replicas": 0 +} + +LOGGING = { + "version": 1, + "formatters": { + "message": { + "format": "%(levelname)s %(asctime)s %(module)s %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + }, + }, + "loggers": { + "": { + "handlers": [ + "console" + ], + "level": "DEBUG", + "propagate": True + }, + "django_elastic_migrations": { + "handlers": ["console"], + "level": "DEBUG", + "propagate": False + }, + }, +} +logging_config.dictConfig(LOGGING) + +# logger = logging.getLogger(__name__) +# logger.debug("using cwd {}".format(root())) +# logger.debug("using python path: {}".format(sys.path)) + +DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = "tests.es_config.ES_CLIENT" +DJANGO_ELASTIC_MIGRATIONS_RECREATE_CONNECTIONS = "tests.es_config.dem_recreate_service_connections" +DJANGO_ELASTIC_MIGRATIONS_CLOSE_CONNECTIONS = "tests.es_config.dem_close_service_connections" +DJANGO_ELASTIC_MIGRATIONS_INDEXES = [ + "tests.search.MovieSearchIndex", +] + +DJANGO_ELASTIC_MIGRATIONS_ENVIRONMENT_PREFIX = "test_" diff --git a/test_utils/__init__.py b/test_utils/__init__.py deleted file mode 100644 index d1fc6f5..0000000 --- a/test_utils/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Test utilities. - -Since py.test discourages putting __init__.py into test directory (i.e. making tests a package) -one cannot import from anywhere under tests folder. However, some utility classes/methods might be useful -in multiple test modules (i.e. factoryboy factories, base test classes). So this package is the place to put them. -""" -from __future__ import print_function diff --git a/CHANGELOG.md b/tests/__init__.py similarity index 100% rename from CHANGELOG.md rename to tests/__init__.py diff --git a/tests/es_config.py b/tests/es_config.py new file mode 100644 index 0000000..6044c19 --- /dev/null +++ b/tests/es_config.py @@ -0,0 +1,55 @@ +# coding=utf-8 +from __future__ import (absolute_import, division, print_function, unicode_literals) +from django.conf import settings +from elasticsearch_dsl import connections +from django.core.cache import caches +from django import db + +ES_CLIENT = None + + +def create_es_client(): + """ + Specified this way so as to facilitate closing and recreating in multiprocessing environments; + see below + :return: + """ + global ES_CLIENT + ES_CLIENT = connections.create_connection(**settings.ELASTICSEARCH_PARAMS) + + +create_es_client() + + +######################################## +# DJANGO ELASTICSEARCH MIGRATIONS CONFIGURATION ↓ +######################################## + + +def dem_close_service_connections(): + """ + Close all connections before spawning multiprocesses indexing. + Only called during `es_update --workers` for parallel indexing. + Connections need to manually opened and closed so that threads + don't reuse the same connection. + https://stackoverflow.com/questions/8242837/django-multiprocessing-and-database-connections + """ + + # close db connections, they will be recreated automatically + db.connections.close_all() + + # close ES connection, needs to be manually recreated + connections.connections.remove_connection("default") + + # close redis connections, will be recreated automatically + for k in settings.CACHES.keys(): + caches[k].close() + + +def dem_recreate_service_connections(): + """ + Recreate all connections inside spawned multiprocessing worker. + Django does this automatically with redis and mysql, but we need + to recreate elasticsearch connection there. + """ + create_es_client() diff --git a/tests/models.py b/tests/models.py new file mode 100644 index 0000000..e6647db --- /dev/null +++ b/tests/models.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +from __future__ import (absolute_import, division, print_function, unicode_literals) + +from django.db import models +from django.utils.encoding import python_2_unicode_compatible + + +@python_2_unicode_compatible +class Movie(models.Model): + title = models.CharField(max_length=500) + year = models.IntegerField() + runtime = models.IntegerField() + genre = models.CharField(max_length=500) + director = models.CharField(max_length=500) + writer = models.CharField(max_length=500) + actors = models.TextField() + plot = models.TextField() + production = models.CharField(max_length=500) + last_modified = models.DateTimeField(auto_now=True, null=True) + + def __str__(self): + return "=1.8,<1.9 + django19: Django>=1.9,<1.10 + django110: Django>=1.10,<1.11 django111: Django>=1.11,<2.0 - django20: Django>=2.0,<2.1 + django2: Django>=2.0,<2.1 + ; https://pypi.org/project/elasticsearch-dsl/#history + es60: elasticsearch-dsl>=6.0.0,<6.1.0 + es61: elasticsearch-dsl>=6.1.0,<6.2.0 + es62: elasticsearch-dsl>=6.2.0,<6.3.0 -r{toxinidir}/requirements/test.txt + commands = - py.test {posargs} + ;https://docs.djangoproject.com/en/2.0/ref/django-admin/#cmdoption-migrate-run-syncdb + ./manage.py migrate --run-syncdb + ./manage.py test {posargs} [testenv:docs] setenv = @@ -48,19 +64,18 @@ commands = python setup.py check --restructuredtext --strict [testenv:quality] +setenv = + PYTHONIOENCODING=utf-8 whitelist_externals = make - rm touch deps = -r{toxinidir}/requirements/quality.txt -r{toxinidir}/requirements/test.txt commands = - touch tests/__init__.py - pylint django_elastic_migrations tests test_utils - pylint --py3k django_elastic_migrations tests test_utils - rm tests/__init__.py + pylint django_elastic_migrations tests + pylint --py3k django_elastic_migrations tests pycodestyle django_elastic_migrations tests pydocstyle django_elastic_migrations tests - isort --check-only --recursive tests test_utils django_elastic_migrations manage.py setup.py test_settings.py + isort --check-only --recursive tests django_elastic_migrations manage.py setup.py test_settings.py make selfcheck