From 3ec2382596ec4ec675b78e3b0716f31435102647 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 06:00:12 -0400 Subject: [PATCH 01/24] myer/add basic test structure for python 2 #2 --- .travis.yml | 38 +++++++ Makefile | 18 ++-- README.md | 19 +++- codecov.yml | 2 + django_elastic_migrations/__init__.py | 2 +- django_elastic_migrations/indexes.py | 13 ++- django_elastic_migrations/utils/test_utils.py | 14 +++ docs/docker_setup.rst | 59 +++++++++++ local.yml | 43 ++++++++ requirements/base.in | 5 +- requirements/dev.txt | 34 ++++--- requirements/quality.txt | 8 +- requirements/test.in | 4 +- requirements/test.txt | 26 +++-- runtests.py | 62 ++++++++++++ setup.py | 9 +- test_settings.py | 48 ++++++++- tests/es_config.py | 55 +++++++++++ tests/models.py | 22 +++++ tests/search.py | 98 +++++++++++++++++++ tests/test_indexes.py | 36 +++++++ tests/test_models.py | 10 -- tests/tests_initial.json | 30 ++++++ tox.ini | 25 +++-- 24 files changed, 602 insertions(+), 78 deletions(-) create mode 100644 .travis.yml create mode 100644 django_elastic_migrations/utils/test_utils.py create mode 100644 docs/docker_setup.rst create mode 100644 local.yml create mode 100644 runtests.py create mode 100644 tests/es_config.py create mode 100644 tests/models.py create mode 100644 tests/search.py create mode 100644 tests/test_indexes.py delete mode 100644 tests/test_models.py create mode 100644 tests/tests_initial.json diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..1ca4a6a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,38 @@ +# Config file for automatic testing at travis-ci.org + +language: python +python: 3.6 + +dist: trusty + +matrix: + include: + - env: TOX_ENV=py27-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + python: 2.7 + - env: TOX_ENV=py27-django19-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + python: 2.7 + - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + python: 3.6 + - env: TOX_ENV=py36-django18-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + python: 3.6 + +before_install: + # work around https://github.com/travis-ci/travis-ci/issues/8363 + - pip install codecov + - wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + - echo "deb $ES_APT_URL stable main" | sudo tee -a /etc/apt/sources.list.d/elk.list + - sudo apt-get update && sudo apt-get install elasticsearch -y + - sudo service elasticsearch start + +# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors +install: pip install -r requirements_test.txt + +# sleep for elasticsearch +before_script: + - sleep 10 + +# command to run tests using coverage, e.g. python setup.py test +script: tox -e $TOX_ENV -- --elasticsearch + +after_success: + - codecov -e $TOX_ENV \ No newline at end of file diff --git a/Makefile b/Makefile index 48e10ad..50c21cd 100644 --- a/Makefile +++ b/Makefile @@ -29,17 +29,13 @@ clean: ## remove generated byte code, coverage reports, and build artifacts rm -fr dist/ rm -fr *.egg-info -coverage: clean ## generate and view HTML coverage report - py.test --cov-report html - $(BROWSER) htmlcov/index.html - upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in pip install -q pip-tools pip-compile --upgrade -o requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in pip-compile --upgrade -o requirements/quality.txt requirements/quality.in pip-compile --upgrade -o requirements/test.txt requirements/base.in requirements/test.in - # Let tox control the Django version for tests - sed '/django==/d' requirements/test.txt > requirements/test.tmp + # Let tox control the Django and elasticsearch-dsl version for tests + sed '/django==/d' requirements/test.txt | sed '/elasticsearch-dsl==/d' | sed '/elasticsearch==/d' > requirements/test.tmp mv requirements/test.tmp requirements/test.txt quality: ## check coding style with pycodestyle and pylint @@ -49,14 +45,20 @@ requirements: ## install development environment requirements pip install -qr requirements/dev.txt --exists-action w pip-sync requirements/dev.txt requirements/test.txt +coverage: clean ## check code coverage quickly with the default Python + coverage run ./runtests.py tests + coverage report -m + coverage html + $(BROWSER) htmlcov/index.html + test: clean ## run tests in the current virtualenv - py.test + python ./runtests.py diff_cover: test diff-cover coverage.xml test-all: ## run tests on every supported Python/Django combination - tox -e quality + # tox -e quality tox validate: quality test ## run tests and quality checks diff --git a/README.md b/README.md index 5b084c6..dc28418 100644 --- a/README.md +++ b/README.md @@ -275,13 +275,28 @@ and return the kwargs you would like to customize. This project uses `make` to manage the build process. Type `make help` to see the available `make` targets. +### Elasticsearch Docker Compose + +`docker-compose -f local.yml up` + +[See docs/docker_setup for more info](./docs/docker_setup.rst) + + ### Requirements +*`make upgrade`* upgrades the dependencies of the requirements to latest +version. This process also excludes `django` and `elasticsearch-dsl` +from the `requirements/test.txt` so they can be injected with different +versions by tox during matrix testing. -Then, `make requirements` runs the pip install. +*`make requirements`* runs the pip install. This project also uses [`pip-tools`](https://github.com/jazzband/pip-tools). The `requirements.txt` files are generated and pinned to latest versions -with `make upgrade`. +with `make upgrade`. + +### Local Tests + +Run `/.manage.py test --settings=bcore.settings ### Updating Egg Info diff --git a/codecov.yml b/codecov.yml index 4da4768..be7cafb 100644 --- a/codecov.yml +++ b/codecov.yml @@ -8,5 +8,7 @@ coverage: default: enabled: yes target: 100% + ignore: + - "tests/*.py" comment: false diff --git a/django_elastic_migrations/__init__.py b/django_elastic_migrations/__init__.py index 7bda52c..49cf695 100644 --- a/django_elastic_migrations/__init__.py +++ b/django_elastic_migrations/__init__.py @@ -59,7 +59,7 @@ es_test_prefix = "test_" -if 'test' in sys.argv: +if 'test' in sys.argv and not environment_prefix == es_test_prefix: environment_prefix = '{}{}'.format(es_test_prefix, environment_prefix) from django_elastic_migrations import apps, indexes diff --git a/django_elastic_migrations/indexes.py b/django_elastic_migrations/indexes.py index 8fdd32c..1f98ac3 100644 --- a/django_elastic_migrations/indexes.py +++ b/django_elastic_migrations/indexes.py @@ -3,6 +3,7 @@ import sys +import django from django.db import ProgrammingError from elasticsearch import TransportError from elasticsearch.helpers import expand_action, bulk @@ -104,7 +105,7 @@ def create_index_model(cls, base_name): try: index_model = DEMIndexModel.objects.create(name=base_name) cls.index_models[base_name] = index_model - except ProgrammingError: + except (ProgrammingError, django.db.utils.OperationalError): # the app is starting up and the database isn't available pass @@ -251,7 +252,7 @@ def update_index_models(cls): from django_elastic_migrations.models import Index as DEMIndexModel try: cls.index_models = {i.name: i for i in DEMIndexModel.objects.all()} - except ProgrammingError: + except (ProgrammingError, django.db.utils.OperationalError): # the app is starting up and the database isn't available pass @@ -727,7 +728,7 @@ def __init__(self, name, using=es_client, version_id=None): :param using: the elasticsearch client to use """ prefixed_name = "{}{}".format(environment_prefix, name) - super(DEMIndex, self).__init__(prefixed_name, using) + super(DEMIndex, self).__init__(prefixed_name, using=using) self.__prefixed_name = prefixed_name self.__base_name = name self.__doc_type = None @@ -892,9 +893,11 @@ def hash_matches(self, their_index_hash): our_index_hash, _ = self.get_index_hash_and_json() return our_index_hash == their_index_hash - def save(self): + def save(self, using=None): + if using is None: + using = es_client try: - super(DEMIndex, self).save() + super(DEMIndex, self).save(using=using) except ValueError as ex: if "Empty value" in ex.message and not self.get_active_version_index_name(): msg = ( diff --git a/django_elastic_migrations/utils/test_utils.py b/django_elastic_migrations/utils/test_utils.py new file mode 100644 index 0000000..18b4f36 --- /dev/null +++ b/django_elastic_migrations/utils/test_utils.py @@ -0,0 +1,14 @@ +from django.test import TestCase + +from django_elastic_migrations import DEMIndexManager + + +class DEMTestCase(TestCase): + + def setUp(self): + DEMIndexManager.test_pre_setup() + super(DEMTestCase, self)._pre_setup() + + def tearDown(self): + DEMIndexManager.test_post_teardown() + super(DEMTestCase, self).tearDown() diff --git a/docs/docker_setup.rst b/docs/docker_setup.rst new file mode 100644 index 0000000..42810ce --- /dev/null +++ b/docs/docker_setup.rst @@ -0,0 +1,59 @@ +Getting Elasticsearch Up and Running Locally With Docker +======================================================== + +.. index:: Docker + +The steps below will get you up and running elasticsearch in a local development environment. +All of these commands assume you are in the project root. + + +Prerequisites +------------- + +* Docker; if you don't have it yet, follow the `installation instructions`_; +* Docker Compose; refer to the official documentation for the `installation guide`_. + +.. _`installation instructions`: https://docs.docker.com/install/#supported-platforms +.. _`installation guide`: https://docs.docker.com/compose/install/ + + +Build the Stack (currently unnecessary) +--------------------------------------- + +Currently, the dev stack consists of prebuilt images. In the future, +if custom docker images are added to the compose file, this may be necessary:: + + $ docker-compose -f local.yml build + + +Run the Stack +------------- + +This brings up Elasticsearch. The first time it is run it might take a while to get started, but subsequent runs will occur quickly. + +Open a terminal at the project root and run the following for local development:: + + $ docker-compose -f local.yml up + +You can also set the environment variable ``COMPOSE_FILE`` pointing to ``local.yml`` like this:: + + $ export COMPOSE_FILE=local.yml + +And then run:: + + $ docker-compose up + +To run in a detached (background) mode, just:: + + $ docker-compose up -d + + +Tips & Tricks +------------- + +Activate a Docker Machine +~~~~~~~~~~~~~~~~~~~~~~~~~ + +This tells our computer that all future commands are specifically for the dev1 machine. Using the ``eval`` command we can switch machines as needed.:: + + $ eval "$(docker-machine env dev1)" diff --git a/local.yml b/local.yml new file mode 100644 index 0000000..c80ff43 --- /dev/null +++ b/local.yml @@ -0,0 +1,43 @@ +# creates a dev elasticsearch (& Kibana, if you want) +# start: +# docker-compose up +# stop: +# docker-compose down +# delete: +# docker-compose kill + +version: '2.1' + +services: + elastic: + image: docker.elastic.co/elasticsearch/elasticsearch:6.0.1 + ports: + - 9200:9200 + - 9300:9300 + environment: + - "discovery.type=single-node" + - "transport.host=127.0.0.1" + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9200/_count"] + interval: 10s + timeout: 10s + retries: 10 + +# uncomment to get kibana, which can be useful for analyzing index contents +# this is currently not a required part of testing infrastructure, just left here for convenience +# kibana: +# image: docker.elastic.co/kibana/kibana:6.0.1 +# depends_on: +# elastic: +# condition: service_healthy +# links: +# - elastic +# ports: +# - 5601:5601 +# environment: +# - "ELASTICSEARCH_URL=http://elastic:9200" +# healthcheck: +# test: ["CMD", "curl", "-f", "http://localhost:5601"] +# interval: 10s +# timeout: 10s +# retries: 10 \ No newline at end of file diff --git a/requirements/base.in b/requirements/base.in index bec2246..5e661e0 100644 --- a/requirements/base.in +++ b/requirements/base.in @@ -1,5 +1,6 @@ # Core requirements for using this application -Django==1.9.8 # Web application framework -elasticsearch-dsl # Used for connecting to elasticsearch +Django # Web application framework +elasticsearch-dsl==6.1.0 # For interacting with Elaticsearch texttable # Printing tabular data from management commands +multiprocessing-logging # for logging in ./manage.py es_update --workers \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index 5d72604..1feaac7 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -6,40 +6,42 @@ # argparse==1.4.0 # via caniusepython3 backports.functools-lru-cache==1.5 # via caniusepython3 -caniusepython3==6.0.0 +caniusepython3==7.0.0 certifi==2018.4.16 # via requests chardet==3.0.4 # via requests click==6.7 # via pip-tools configparser==3.5.0 # via pydocstyle -diff-cover==1.0.2 +diff-cover==1.0.4 distlib==0.2.7 # via caniusepython3 -django==1.9.8 +django==1.11.14 elasticsearch-dsl==6.1.0 -elasticsearch==6.2.0 # via elasticsearch-dsl +elasticsearch==6.3.0 # via elasticsearch-dsl first==2.0.1 # via pip-tools futures==3.2.0 # via caniusepython3, isort -idna==2.6 # via requests -inflect==0.2.5 # via jinja2-pluralize +idna==2.7 # via requests +inflect==1.0.0 # via jinja2-pluralize ipaddress==1.0.22 # via elasticsearch-dsl isort==4.3.4 jinja2-pluralize==0.3.0 # via diff-cover jinja2==2.10 # via diff-cover, jinja2-pluralize markupsafe==1.0 # via jinja2 -packaging==17.1 # via caniusepython3 +multiprocessing-logging==0.2.6 +packaging==17.1 # via caniusepython3, tox pip-tools==2.0.2 -pluggy==0.6.0 # via tox -py==1.5.3 # via tox +pluggy==0.7.1 # via tox +py==1.5.4 # via tox pycodestyle==2.4.0 pydocstyle==2.1.1 pygments==2.2.0 # via diff-cover pyparsing==2.2.0 # via packaging -python-dateutil==2.7.2 # via elasticsearch-dsl -requests==2.18.4 # via caniusepython3 +python-dateutil==2.7.3 # via elasticsearch-dsl +pytz==2018.5 # via django +requests==2.19.1 # via caniusepython3 six==1.11.0 # via diff-cover, elasticsearch-dsl, packaging, pip-tools, pydocstyle, python-dateutil, tox snowballstemmer==1.2.1 # via pydocstyle -texttable==1.2.1 +texttable==1.4.0 tox-battery==0.5.1 -tox==3.0.0 -urllib3==1.22 # via elasticsearch, requests -virtualenv==15.2.0 # via tox -wheel==0.31.0 +tox==3.1.2 +urllib3==1.23 # via elasticsearch, requests +virtualenv==16.0.0 # via tox +wheel==0.31.1 diff --git a/requirements/quality.txt b/requirements/quality.txt index 4c441c5..55117fb 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -6,19 +6,19 @@ # argparse==1.4.0 # via caniusepython3 backports.functools-lru-cache==1.5 # via caniusepython3 -caniusepython3==6.0.0 +caniusepython3==7.0.0 certifi==2018.4.16 # via requests chardet==3.0.4 # via requests configparser==3.5.0 # via pydocstyle distlib==0.2.7 # via caniusepython3 futures==3.2.0 # via caniusepython3, isort -idna==2.6 # via requests +idna==2.7 # via requests isort==4.3.4 packaging==17.1 # via caniusepython3 pycodestyle==2.4.0 pydocstyle==2.1.1 pyparsing==2.2.0 # via packaging -requests==2.18.4 # via caniusepython3 +requests==2.19.1 # via caniusepython3 six==1.11.0 # via packaging, pydocstyle snowballstemmer==1.2.1 # via pydocstyle -urllib3==1.22 # via requests +urllib3==1.23 # via requests diff --git a/requirements/test.in b/requirements/test.in index 9758da5..fcf8ac9 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -1,4 +1,4 @@ # Requirements for test runs. -pytest-cov # pytest extension for code coverage statistics -pytest-django # pytest extension for better Django support +codecov # for https://codecov.io/ github integration +coverage # for quick, one-off coverage tests \ No newline at end of file diff --git a/requirements/test.txt b/requirements/test.txt index 27e76a9..c345c02 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -4,18 +4,16 @@ # # pip-compile --output-file requirements/test.txt requirements/base.in requirements/test.in # -attrs==17.4.0 # via pytest -coverage==4.5.1 # via pytest-cov -elasticsearch-dsl==6.1.0 -elasticsearch==6.2.0 # via elasticsearch-dsl -funcsigs==1.0.2 # via pytest +certifi==2018.4.16 # via requests +chardet==3.0.4 # via requests +codecov==2.0.15 +coverage==4.5.1 +idna==2.7 # via requests ipaddress==1.0.22 # via elasticsearch-dsl -more-itertools==4.1.0 # via pytest -pluggy==0.6.0 # via pytest -py==1.5.3 # via pytest -pytest-cov==2.5.1 -pytest==3.5.1 # via pytest-cov, pytest-django -python-dateutil==2.7.2 # via elasticsearch-dsl -six==1.11.0 # via elasticsearch-dsl, more-itertools, pytest, python-dateutil -texttable==1.2.1 -urllib3==1.22 # via elasticsearch +multiprocessing-logging==0.2.6 +python-dateutil==2.7.3 # via elasticsearch-dsl +pytz==2018.5 # via django +requests==2.19.1 # via codecov +six==1.11.0 # via elasticsearch-dsl, python-dateutil +texttable==1.4.0 +urllib3==1.23 # via elasticsearch, requests diff --git a/runtests.py b/runtests.py new file mode 100644 index 0000000..5e1b158 --- /dev/null +++ b/runtests.py @@ -0,0 +1,62 @@ +import os +import sys +import argparse + +try: + from django.conf import settings, global_settings + from django.test.utils import get_runner + + def get_settings(): + import test_settings + settings.configure(global_settings, **test_settings.__dict__) + + # in newer versions of django you have to call .setup() directly + try: + import django + setup = django.setup + except AttributeError: + pass + else: + setup() + + return settings + +except ImportError: + import traceback + traceback.print_exc() + msg = "To fix this error, run: pip install -r requirements_test.txt or make sure your virtualenv is activated" + raise ImportError(msg) + + +def make_parser(): + parser = argparse.ArgumentParser() + parser.add_argument( + '--elasticsearch', + nargs='?', + metavar='localhost:9200', + const='localhost:9200', + help="To run integration test against an Elasticsearch server", + ) + return parser + + +def run_tests(*test_args): + args, test_args = make_parser().parse_known_args(test_args) + if args.elasticsearch: + os.environ.setdefault('ELASTICSEARCH_URL', args.elasticsearch) + + if not test_args: + test_args = ['tests'] + + settings = get_settings() + TestRunner = get_runner(settings) + test_runner = TestRunner() + + failures = test_runner.run_tests(test_args) + + if failures: + sys.exit(bool(failures)) + + +if __name__ == '__main__': + run_tests(*sys.argv[1:]) \ No newline at end of file diff --git a/setup.py b/setup.py index 7bc6f55..c676d9f 100755 --- a/setup.py +++ b/setup.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- # pylint: disable=C0111,W6005,W6100 -from __future__ import print_function from __future__ import absolute_import, print_function import os @@ -33,8 +32,10 @@ def get_version(*file_paths): os.system("git push origin %s" % VERSION) sys.exit() -README = open(os.path.join(os.path.dirname(__file__), 'README.md')).read() -CHANGELOG = open(os.path.join(os.path.dirname(__file__), 'CHANGELOG.md')).read() +# README = open(os.path.join(os.path.dirname(__file__), 'README.md')).read() +# CHANGELOG = open(os.path.join(os.path.dirname(__file__), 'CHANGELOG.md')).read() +README = 'readme' +CHANGELOG = 'changelog' setup( name='django-elastic-migrations', @@ -43,7 +44,7 @@ def get_version(*file_paths): long_description=README + '\n\n' + CHANGELOG, author='Harvard Business School, HBX Department', author_email='pnore@hbs.edu', - url='https://github.com/HBS-HBX/django_elastic_migrations', + url='https://github.com/HBS-HBX/django-elastic-migrations', packages=[ 'django_elastic_migrations', ], diff --git a/test_settings.py b/test_settings.py index 5c7f1ea..c521eab 100644 --- a/test_settings.py +++ b/test_settings.py @@ -8,8 +8,13 @@ from __future__ import print_function from __future__ import absolute_import, unicode_literals +import subprocess + +import django from os.path import abspath, dirname, join +DEBUG = True + def root(*args): """ @@ -33,6 +38,7 @@ def root(*args): 'django.contrib.auth', 'django.contrib.contenttypes', 'django_elastic_migrations', + 'tests' ) ROOT_URLCONF = 'django_elastic_migrations.urls' @@ -48,4 +54,44 @@ def root(*args): ], } -DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = 'django_elastic_migrations.utils.es_utils.DEFAULT_ES_CLIENT' +ELASTICSEARCH_INDEX_SETTINGS = { + "number_of_shards": 1, + "number_of_replicas": 0 +} + +LOGGING = { + "version": 1, + "formatters": { + "message": { + "format": "%(levelname)s %(asctime)s %(module)s %(message)s" + } + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + }, + }, + "loggers": { + "": { + "handlers": [ + "console" + ], + "level": "INFO", + "propagate": True + }, + "django_elastic_migrations": { + "handlers": ["console"], + "level": "DEBUG", + "propagate": False + }, + }, +} + +DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = "tests.es_config.ES_CLIENT" +DJANGO_ELASTIC_MIGRATIONS_RECREATE_CONNECTIONS = "tests.es_config.dem_recreate_service_connections" +DJANGO_ELASTIC_MIGRATIONS_CLOSE_CONNECTIONS = "tests.es_config.dem_close_service_connections" +DJANGO_ELASTIC_MIGRATIONS_INDEXES = [ + "tests.search.MovieSearchIndex", +] + +DJANGO_ELASTIC_MIGRATIONS_ENVIRONMENT_PREFIX = "test_" diff --git a/tests/es_config.py b/tests/es_config.py new file mode 100644 index 0000000..c5788fb --- /dev/null +++ b/tests/es_config.py @@ -0,0 +1,55 @@ +# coding=utf-8 +from __future__ import print_function +from django.conf import settings +from elasticsearch_dsl import connections +from django.core.cache import caches +from django import db + +ES_CLIENT = None + + +def create_es_client(): + """ + Specified this way so as to facilitate closing and recreating in multiprocessing environments; + see below + :return: + """ + global ES_CLIENT + ES_CLIENT = connections.create_connection(**settings.ELASTICSEARCH_PARAMS) + + +create_es_client() + + +######################################## +# DJANGO ELASTICSEARCH MIGRATIONS CONFIGURATION ↓ +######################################## + + +def dem_close_service_connections(): + """ + Close all connections before spawning multiprocesses indexing. + Only called during `es_update --workers` for parallel indexing. + Connections need to manually opened and closed so that threads + don't reuse the same connection. + https://stackoverflow.com/questions/8242837/django-multiprocessing-and-database-connections + """ + + # close db connections, they will be recreated automatically + db.connections.close_all() + + # close ES connection, needs to be manually recreated + connections.connections.remove_connection("default") + + # close redis connections, will be recreated automatically + for k in settings.CACHES.keys(): + caches[k].close() + + +def dem_recreate_service_connections(): + """ + Recreate all connections inside spawned multiprocessing worker. + Django does this automatically with redis and mysql, but we need + to recreate elasticsearch connection there. + """ + create_es_client() diff --git a/tests/models.py b/tests/models.py new file mode 100644 index 0000000..6a1700a --- /dev/null +++ b/tests/models.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import models +from django.utils.encoding import python_2_unicode_compatible + + +@python_2_unicode_compatible +class Movie(models.Model): + title = models.CharField(max_length=500) + year = models.IntegerField() + runtime = models.IntegerField() + genre = models.CharField(max_length=500) + director = models.CharField(max_length=500) + writer = models.CharField(max_length=500) + actors = models.TextField() + plot = models.TextField() + production = models.CharField(max_length=500) + last_modified = models.DateTimeField(auto_now=True, null=True) + + def __str__(self): + return "=1.8,<1.9 + django19: Django>=1.9,<1.10 + django110: Django>=1.10,<1.11 django111: Django>=1.11,<2.0 - django20: Django>=2.0,<2.1 + django2: Django>=2.0,<2.1 + ; https://pypi.org/project/elasticsearch-dsl/#history + es60: elasticsearch-dsl>=6.0.0,<6.1.0 + es61: elasticsearch-dsl>=6.1.0,<6.2.0 + es62: elasticsearch-dsl>=6.2.0,<6.3.0 -r{toxinidir}/requirements/test.txt -commands = - py.test {posargs} + +commands = python runtests.py {posargs} [testenv:docs] setenv = From 4d4b19817c2504c6f62efed03bdefb7c6061bf5f Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 06:03:42 -0400 Subject: [PATCH 02/24] remove py3 tests temporarily until travis is working myer/add basic test structure for python 2 #2 --- .travis.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1ca4a6a..d084914 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,11 @@ matrix: python: 2.7 - env: TOX_ENV=py27-django19-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 2.7 - - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt - python: 3.6 - - env: TOX_ENV=py36-django18-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt - python: 3.6 +# TBD Python 3 tests ... +# - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt +# python: 3.6 +# - env: TOX_ENV=py36-django18-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt +# python: 3.6 before_install: # work around https://github.com/travis-ci/travis-ci/issues/8363 From cbff6a5e74b17176303b5351fa9af116e2a12249 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 06:13:41 -0400 Subject: [PATCH 03/24] add build status button to readme myer/add basic test structure for python 2 #2 --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index dc28418..3094d63 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,9 @@ # Django Elastic Migrations Django Elastic Migrations provides a way to control the deployment of -multiple Elasticsearch schemas over time. +multiple Elasticsearch schemas over time. + +[![Build Status](https://travis-ci.com/HBS-HBX/django-elastic-migrations.svg?branch=master)](https://travis-ci.com/HBS-HBX/django-elastic-migrations) ## Overview From 51882be844ed2aeb037aafb04721fd01fdcd19b8 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 06:18:12 -0400 Subject: [PATCH 04/24] update link to requirements file myer/add basic test structure for python 2 #2 --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index d084914..04ab672 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,8 +25,8 @@ before_install: - sudo apt-get update && sudo apt-get install elasticsearch -y - sudo service elasticsearch start -# command to install dependencies, e.g. pip install -r requirements.txt --use-mirrors -install: pip install -r requirements_test.txt +# command to install dependencies, e.g. pip install -r requirements.txt +install: pip install -r requirements/test.txt # sleep for elasticsearch before_script: From 96edf09183ceae5dba7939c6b689cb16d3f7f53a Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 06:23:25 -0400 Subject: [PATCH 05/24] add tox to `requirements/test.txt` myer/add basic test structure for python 2 #2 --- requirements/test.in | 4 +++- requirements/test.txt | 9 ++++++++- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/requirements/test.in b/requirements/test.in index fcf8ac9..b2c5612 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -1,4 +1,6 @@ # Requirements for test runs. codecov # for https://codecov.io/ github integration -coverage # for quick, one-off coverage tests \ No newline at end of file +coverage # for quick, one-off coverage tests +tox # virtualenv management for tests +tox-battery # Makes tox aware of requirements file changes diff --git a/requirements/test.txt b/requirements/test.txt index c345c02..a4f2b0c 100644 --- a/requirements/test.txt +++ b/requirements/test.txt @@ -11,9 +11,16 @@ coverage==4.5.1 idna==2.7 # via requests ipaddress==1.0.22 # via elasticsearch-dsl multiprocessing-logging==0.2.6 +packaging==17.1 # via tox +pluggy==0.7.1 # via tox +py==1.5.4 # via tox +pyparsing==2.2.0 # via packaging python-dateutil==2.7.3 # via elasticsearch-dsl pytz==2018.5 # via django requests==2.19.1 # via codecov -six==1.11.0 # via elasticsearch-dsl, python-dateutil +six==1.11.0 # via elasticsearch-dsl, packaging, python-dateutil, tox texttable==1.4.0 +tox-battery==0.5.1 +tox==3.1.2 urllib3==1.23 # via elasticsearch, requests +virtualenv==16.0.0 # via tox From 36dc2cf312bcaef564f693faf0d3deaa07152ff3 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 06:35:45 -0400 Subject: [PATCH 06/24] remove superfluous parameters myer/add basic test structure for python 2 #2 --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 04ab672..80838ea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,15 +25,15 @@ before_install: - sudo apt-get update && sudo apt-get install elasticsearch -y - sudo service elasticsearch start -# command to install dependencies, e.g. pip install -r requirements.txt +# command to install dependencies (e.g. tox) install: pip install -r requirements/test.txt # sleep for elasticsearch before_script: - sleep 10 -# command to run tests using coverage, e.g. python setup.py test -script: tox -e $TOX_ENV -- --elasticsearch +# command to run tests +script: tox -e $TOX_ENV after_success: - codecov -e $TOX_ENV \ No newline at end of file From 857c11dbb69cfb0c735246589093c8e1258cbbe1 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 09:10:20 -0400 Subject: [PATCH 07/24] add pythonpath to tox.ini and update manifest to match file names myer/add basic test structure for python 2 #2 --- .coveragerc | 2 +- CHANGELOG.md => CHANGELOG.rst | 0 MANIFEST.in | 2 +- setup.py | 6 ++---- tox.ini | 6 ++++++ 5 files changed, 10 insertions(+), 6 deletions(-) rename CHANGELOG.md => CHANGELOG.rst (100%) diff --git a/.coveragerc b/.coveragerc index ff01f55..fb91fb2 100644 --- a/.coveragerc +++ b/.coveragerc @@ -4,7 +4,7 @@ data_file = .coverage source=django_elastic_migrations omit = test_settings - *migrations* + */migrations/* *admin.py *static* *templates* diff --git a/CHANGELOG.md b/CHANGELOG.rst similarity index 100% rename from CHANGELOG.md rename to CHANGELOG.rst diff --git a/MANIFEST.in b/MANIFEST.in index 1450764..dc582f6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,5 +2,5 @@ include AUTHORS include CHANGELOG.rst include CONTRIBUTING.rst include LICENSE.txt -include README.rst +include README.md recursive-include django_elastic_migrations *.html *.png *.gif *js *.css *jpg *jpeg *svg *py diff --git a/setup.py b/setup.py index c676d9f..596c343 100755 --- a/setup.py +++ b/setup.py @@ -32,10 +32,8 @@ def get_version(*file_paths): os.system("git push origin %s" % VERSION) sys.exit() -# README = open(os.path.join(os.path.dirname(__file__), 'README.md')).read() -# CHANGELOG = open(os.path.join(os.path.dirname(__file__), 'CHANGELOG.md')).read() -README = 'readme' -CHANGELOG = 'changelog' +README = open('README.md').read() +CHANGELOG = open('CHANGELOG.rst').read() setup( name='django-elastic-migrations', diff --git a/tox.ini b/tox.ini index 5e751d4..3da2612 100644 --- a/tox.ini +++ b/tox.ini @@ -19,9 +19,15 @@ max-line-length = 120 ignore = D101,D200,D203,D212 match-dir = (?!migrations) +basepython = + py27: python2.7 + py35: python3.5 + py36: python3.6 + [testenv] setenv = PYTHONIOENCODING=utf-8 + PYTHONPATH = {toxinidir}/django_elastic_migrations deps = django18: Django>=1.8,<1.9 From 1e02a7f8b9cd74a04c6447735cf592bc2567441c Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 09:30:21 -0400 Subject: [PATCH 08/24] add travis requirements file myer/add basic test structure for python 2 #2 --- .travis.yml | 3 ++- Makefile | 1 + django_elastic_migrations/__init__.py | 2 +- requirements/test.in | 2 +- requirements/travis.in | 5 +++++ requirements/travis.txt | 22 ++++++++++++++++++++++ test_settings.py | 1 - 7 files changed, 32 insertions(+), 4 deletions(-) create mode 100644 requirements/travis.in create mode 100644 requirements/travis.txt diff --git a/.travis.yml b/.travis.yml index 80838ea..47788fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,6 +18,7 @@ matrix: # python: 3.6 before_install: + - pip install --upgrade pip # work around https://github.com/travis-ci/travis-ci/issues/8363 - pip install codecov - wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - @@ -26,7 +27,7 @@ before_install: - sudo service elasticsearch start # command to install dependencies (e.g. tox) -install: pip install -r requirements/test.txt +install: pip install -r requirements/travis.txt # sleep for elasticsearch before_script: diff --git a/Makefile b/Makefile index 50c21cd..d695e2e 100644 --- a/Makefile +++ b/Makefile @@ -34,6 +34,7 @@ upgrade: ## update the requirements/*.txt files with the latest packages satisfy pip-compile --upgrade -o requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in pip-compile --upgrade -o requirements/quality.txt requirements/quality.in pip-compile --upgrade -o requirements/test.txt requirements/base.in requirements/test.in + pip-compile --upgrade -o requirements/travis.txt requirements/travis.in # Let tox control the Django and elasticsearch-dsl version for tests sed '/django==/d' requirements/test.txt | sed '/elasticsearch-dsl==/d' | sed '/elasticsearch==/d' > requirements/test.tmp mv requirements/test.tmp requirements/test.txt diff --git a/django_elastic_migrations/__init__.py b/django_elastic_migrations/__init__.py index 49cf695..d876f2e 100644 --- a/django_elastic_migrations/__init__.py +++ b/django_elastic_migrations/__init__.py @@ -25,7 +25,7 @@ 'This should be the python path to the elasticsearch client ' 'to use for indexing.') - +logger.debug("using DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = {}".format(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT)) es_client = loading.import_module_element(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT) diff --git a/requirements/test.in b/requirements/test.in index b2c5612..a3e1454 100644 --- a/requirements/test.in +++ b/requirements/test.in @@ -3,4 +3,4 @@ codecov # for https://codecov.io/ github integration coverage # for quick, one-off coverage tests tox # virtualenv management for tests -tox-battery # Makes tox aware of requirements file changes +tox-battery # Makes tox aware of requirements file changes \ No newline at end of file diff --git a/requirements/travis.in b/requirements/travis.in new file mode 100644 index 0000000..f1f15dc --- /dev/null +++ b/requirements/travis.in @@ -0,0 +1,5 @@ +# Requirements for running tests in Travis + +codecov # Code coverage reporting +tox # Virtualenv management for tests +tox-battery # Makes tox aware of requirements file changes \ No newline at end of file diff --git a/requirements/travis.txt b/requirements/travis.txt new file mode 100644 index 0000000..de4f450 --- /dev/null +++ b/requirements/travis.txt @@ -0,0 +1,22 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file requirements/travis.txt requirements/travis.in +# +certifi==2018.4.16 # via requests +chardet==3.0.4 # via requests +codecov==2.0.15 +coverage==4.5.1 # via codecov +idna==2.7 # via requests +packaging==17.1 # via tox +pluggy==0.7.1 # via tox +py==1.5.4 # via tox +pyparsing==2.2.0 # via packaging +requests==2.19.1 # via codecov +six==1.11.0 # via packaging, tox +tox-battery==0.5.1 +tox==3.1.2 +urllib3==1.23 # via requests +virtualenv==16.0.0 # via tox +-e . \ No newline at end of file diff --git a/test_settings.py b/test_settings.py index c521eab..80d2bf4 100644 --- a/test_settings.py +++ b/test_settings.py @@ -47,7 +47,6 @@ def root(*args): # parameters to pass to base Elasticsearch() instance # https://elasticsearch-py.readthedocs.io/en/master/api.html#elasticsearch -# override in app vars ELASTICSEARCH_PARAMS = { 'hosts': [ 'localhost:9200' From 44ed9b3a41269a4ecdf2b267e03b2bd8499d8fc0 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 10:51:14 -0400 Subject: [PATCH 09/24] add travis requirements file myer/add basic test structure for python 2 #2 --- requirements/travis.txt | 3 +-- runtests.py | 1 + 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/travis.txt b/requirements/travis.txt index de4f450..eb6bdf3 100644 --- a/requirements/travis.txt +++ b/requirements/travis.txt @@ -18,5 +18,4 @@ six==1.11.0 # via packaging, tox tox-battery==0.5.1 tox==3.1.2 urllib3==1.23 # via requests -virtualenv==16.0.0 # via tox --e . \ No newline at end of file +virtualenv==16.0.0 # via tox \ No newline at end of file diff --git a/runtests.py b/runtests.py index 5e1b158..baaccdf 100644 --- a/runtests.py +++ b/runtests.py @@ -17,6 +17,7 @@ def get_settings(): except AttributeError: pass else: + print("Testing in cwd {}".format(os.getcwd())) setup() return settings From 7e0d9482274d8f56179ec286dd5dd5abda6ab7b9 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 13:13:27 -0400 Subject: [PATCH 10/24] use normal django tests myer/add basic test structure for python 2 #2 --- Makefile | 4 ++-- runtests.py | 63 ----------------------------------------------------- tox.ini | 2 +- 3 files changed, 3 insertions(+), 66 deletions(-) delete mode 100644 runtests.py diff --git a/Makefile b/Makefile index d695e2e..412992f 100644 --- a/Makefile +++ b/Makefile @@ -47,13 +47,13 @@ requirements: ## install development environment requirements pip-sync requirements/dev.txt requirements/test.txt coverage: clean ## check code coverage quickly with the default Python - coverage run ./runtests.py tests + coverage run ./manage.py test coverage report -m coverage html $(BROWSER) htmlcov/index.html test: clean ## run tests in the current virtualenv - python ./runtests.py + ./manage.py test diff_cover: test diff-cover coverage.xml diff --git a/runtests.py b/runtests.py deleted file mode 100644 index baaccdf..0000000 --- a/runtests.py +++ /dev/null @@ -1,63 +0,0 @@ -import os -import sys -import argparse - -try: - from django.conf import settings, global_settings - from django.test.utils import get_runner - - def get_settings(): - import test_settings - settings.configure(global_settings, **test_settings.__dict__) - - # in newer versions of django you have to call .setup() directly - try: - import django - setup = django.setup - except AttributeError: - pass - else: - print("Testing in cwd {}".format(os.getcwd())) - setup() - - return settings - -except ImportError: - import traceback - traceback.print_exc() - msg = "To fix this error, run: pip install -r requirements_test.txt or make sure your virtualenv is activated" - raise ImportError(msg) - - -def make_parser(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--elasticsearch', - nargs='?', - metavar='localhost:9200', - const='localhost:9200', - help="To run integration test against an Elasticsearch server", - ) - return parser - - -def run_tests(*test_args): - args, test_args = make_parser().parse_known_args(test_args) - if args.elasticsearch: - os.environ.setdefault('ELASTICSEARCH_URL', args.elasticsearch) - - if not test_args: - test_args = ['tests'] - - settings = get_settings() - TestRunner = get_runner(settings) - test_runner = TestRunner() - - failures = test_runner.run_tests(test_args) - - if failures: - sys.exit(bool(failures)) - - -if __name__ == '__main__': - run_tests(*sys.argv[1:]) \ No newline at end of file diff --git a/tox.ini b/tox.ini index 3da2612..9213e4b 100644 --- a/tox.ini +++ b/tox.ini @@ -41,7 +41,7 @@ deps = es62: elasticsearch-dsl>=6.2.0,<6.3.0 -r{toxinidir}/requirements/test.txt -commands = python runtests.py {posargs} +commands = python manage.py test {posargs} [testenv:docs] setenv = From 9cb2c5c602b1b8b4653079f90a2f87ebd81d53c9 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 15:25:01 -0400 Subject: [PATCH 11/24] add log handling to __init__.py and rename log handler * update CHANGELOG myer/add basic test structure for python 2 #2 --- .travis.yml | 4 ++-- CHANGELOG.rst | 11 +++++++++++ django_elastic_migrations/__init__.py | 8 ++++++-- .../utils/django_elastic_migrations_log.py | 2 +- test_settings.py | 7 +++++++ tox.ini | 2 +- 6 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 47788fb..2b65b12 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,8 @@ dist: trusty matrix: include: - - env: TOX_ENV=py27-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt - python: 2.7 +# - env: TOX_ENV=py27-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt +# python: 2.7 - env: TOX_ENV=py27-django19-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 2.7 # TBD Python 3 tests ... diff --git a/CHANGELOG.rst b/CHANGELOG.rst index e69de29..f1f6e4d 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -0,0 +1,11 @@ +Changelog +--------- + +0.6.0 (2018-08-01) +~~~~~~~~~~~~~~~~~~ +* Added test structure for py2 - GH #2 +* Renamed default log handler from ``django-elastic-migrations`` to ``django_elastic_migrations`` + +0.5.3 (2018-07-23) +~~~~~~~~~~~~~~~~~~ +* First basic release \ No newline at end of file diff --git a/django_elastic_migrations/__init__.py b/django_elastic_migrations/__init__.py index d876f2e..f5bba17 100644 --- a/django_elastic_migrations/__init__.py +++ b/django_elastic_migrations/__init__.py @@ -10,7 +10,7 @@ from django_elastic_migrations.utils import loading from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger -__version__ = '0.5.3' +__version__ = '0.6.0' default_app_config = 'django_elastic_migrations.apps.DjangoElasticMigrationsConfig' # pylint: disable=invalid-name @@ -26,8 +26,12 @@ 'to use for indexing.') logger.debug("using DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = {}".format(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT)) -es_client = loading.import_module_element(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT) +try: + es_client = loading.import_module_element(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT) +except ImportError: + logger.warning("DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT {} not found: ".format(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT)) + from tests.es_config import ES_CLIENT codebase_id = getattr(settings, 'DJANGO_ELASTIC_MIGRATIONS_GET_CODEBASE_ID', "") diff --git a/django_elastic_migrations/utils/django_elastic_migrations_log.py b/django_elastic_migrations/utils/django_elastic_migrations_log.py index 3e1e569..d092a86 100644 --- a/django_elastic_migrations/utils/django_elastic_migrations_log.py +++ b/django_elastic_migrations/utils/django_elastic_migrations_log.py @@ -6,7 +6,7 @@ install_mp_handler() -def get_logger(name="django-elastic-migrations"): +def get_logger(name="django_elastic_migrations"): real_logger = logging.getLogger(name) for level in ['CRITICAL', 'ERROR', 'WARNING', 'INFO', 'DEBUG']: setattr(real_logger, level, getattr(logging, level)) diff --git a/test_settings.py b/test_settings.py index 80d2bf4..79cd494 100644 --- a/test_settings.py +++ b/test_settings.py @@ -8,6 +8,8 @@ from __future__ import print_function from __future__ import absolute_import, unicode_literals +import logging +from logging import config as logging_config import subprocess import django @@ -85,6 +87,11 @@ def root(*args): }, }, } +logging_config.dictConfig(LOGGING) + +logger = logging.getLogger(__name__) +logger.info("using cwd {}".format(root())) +logger.debug("this is debug msg") DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = "tests.es_config.ES_CLIENT" DJANGO_ELASTIC_MIGRATIONS_RECREATE_CONNECTIONS = "tests.es_config.dem_recreate_service_connections" diff --git a/tox.ini b/tox.ini index 9213e4b..a6f31f2 100644 --- a/tox.ini +++ b/tox.ini @@ -41,7 +41,7 @@ deps = es62: elasticsearch-dsl>=6.2.0,<6.3.0 -r{toxinidir}/requirements/test.txt -commands = python manage.py test {posargs} +commands = ./manage.py test {posargs} [testenv:docs] setenv = From edbb4018c46e651c2840a37afdf18d017c4b46c4 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 15:40:59 -0400 Subject: [PATCH 12/24] add debug logging to settings file myer/add basic test structure for python 2 #2 --- django_elastic_migrations/__init__.py | 2 +- test_settings.py | 12 +++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/django_elastic_migrations/__init__.py b/django_elastic_migrations/__init__.py index f5bba17..faba4d2 100644 --- a/django_elastic_migrations/__init__.py +++ b/django_elastic_migrations/__init__.py @@ -31,7 +31,7 @@ es_client = loading.import_module_element(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT) except ImportError: logger.warning("DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT {} not found: ".format(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT)) - from tests.es_config import ES_CLIENT + from tests.es_config import ES_CLIENT as es_client codebase_id = getattr(settings, 'DJANGO_ELASTIC_MIGRATIONS_GET_CODEBASE_ID', "") diff --git a/test_settings.py b/test_settings.py index 79cd494..9082df0 100644 --- a/test_settings.py +++ b/test_settings.py @@ -9,6 +9,8 @@ from __future__ import absolute_import, unicode_literals import logging +import os +import sys from logging import config as logging_config import subprocess @@ -77,7 +79,7 @@ def root(*args): "handlers": [ "console" ], - "level": "INFO", + "level": "DEBUG", "propagate": True }, "django_elastic_migrations": { @@ -90,8 +92,12 @@ def root(*args): logging_config.dictConfig(LOGGING) logger = logging.getLogger(__name__) -logger.info("using cwd {}".format(root())) -logger.debug("this is debug msg") +logger.debug("using cwd {}".format(root())) +logger.debug("using python path: {}".format(sys.path)) +try: + logger.debug("{}".format([str(p) for p in os.environ['PYTHONPATH'].split(os.pathsep)])) +except KeyError: + logger.debug("os.environ['PYTHONPATH'] wasn't defined") DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = "tests.es_config.ES_CLIENT" DJANGO_ELASTIC_MIGRATIONS_RECREATE_CONNECTIONS = "tests.es_config.dem_recreate_service_connections" From 0ca7c75296602a1f43e63b7850a7045d5168c1ed Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 15:52:53 -0400 Subject: [PATCH 13/24] add debug logging with files in cwd and try removing restriction on pythonpath myer/add basic test structure for python 2 #2 --- test_settings.py | 1 + tox.ini | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/test_settings.py b/test_settings.py index 9082df0..0eda04c 100644 --- a/test_settings.py +++ b/test_settings.py @@ -94,6 +94,7 @@ def root(*args): logger = logging.getLogger(__name__) logger.debug("using cwd {}".format(root())) logger.debug("using python path: {}".format(sys.path)) +logger.debug(os.listdir(root())) try: logger.debug("{}".format([str(p) for p in os.environ['PYTHONPATH'].split(os.pathsep)])) except KeyError: diff --git a/tox.ini b/tox.ini index a6f31f2..c1520a0 100644 --- a/tox.ini +++ b/tox.ini @@ -27,7 +27,7 @@ basepython = [testenv] setenv = PYTHONIOENCODING=utf-8 - PYTHONPATH = {toxinidir}/django_elastic_migrations + PYTHONPATH = {toxinidir} deps = django18: Django>=1.8,<1.9 From d885edb500f61a21f9c6a43f5e6a645133d2597b Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Wed, 1 Aug 2018 16:03:05 -0400 Subject: [PATCH 14/24] remove erroneous gitignore path and add `tests/__init__.py` myer/add basic test structure for python 2 #2 --- .gitignore | 3 --- django_elastic_migrations/__init__.py | 5 +++-- test_settings.py | 11 +++-------- tests/__init__.py | 0 4 files changed, 6 insertions(+), 13 deletions(-) create mode 100644 tests/__init__.py diff --git a/.gitignore b/.gitignore index c776a2d..899966e 100644 --- a/.gitignore +++ b/.gitignore @@ -61,8 +61,5 @@ output/*/index.html requirements/private.in requirements/private.txt -# tox environment temporary artifacts -tests/__init__.py - # Development task artifacts default.db diff --git a/django_elastic_migrations/__init__.py b/django_elastic_migrations/__init__.py index faba4d2..5a33ff6 100644 --- a/django_elastic_migrations/__init__.py +++ b/django_elastic_migrations/__init__.py @@ -30,8 +30,9 @@ try: es_client = loading.import_module_element(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT) except ImportError: - logger.warning("DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT {} not found: ".format(settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT)) - from tests.es_config import ES_CLIENT as es_client + logger.error("DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT {} not found. Please check your python path and django settings ".format( + settings.DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT)) + raise codebase_id = getattr(settings, 'DJANGO_ELASTIC_MIGRATIONS_GET_CODEBASE_ID', "") diff --git a/test_settings.py b/test_settings.py index 0eda04c..4cdb239 100644 --- a/test_settings.py +++ b/test_settings.py @@ -91,14 +91,9 @@ def root(*args): } logging_config.dictConfig(LOGGING) -logger = logging.getLogger(__name__) -logger.debug("using cwd {}".format(root())) -logger.debug("using python path: {}".format(sys.path)) -logger.debug(os.listdir(root())) -try: - logger.debug("{}".format([str(p) for p in os.environ['PYTHONPATH'].split(os.pathsep)])) -except KeyError: - logger.debug("os.environ['PYTHONPATH'] wasn't defined") +# logger = logging.getLogger(__name__) +# logger.debug("using cwd {}".format(root())) +# logger.debug("using python path: {}".format(sys.path)) DJANGO_ELASTIC_MIGRATIONS_ES_CLIENT = "tests.es_config.ES_CLIENT" DJANGO_ELASTIC_MIGRATIONS_RECREATE_CONNECTIONS = "tests.es_config.dem_recreate_service_connections" diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 From 4e1d6d01b7e67355e4774f0d7405c892d112464f Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Thu, 2 Aug 2018 07:13:45 -0400 Subject: [PATCH 15/24] add `./manage.py migrate --run-syncdb` to install tasks for travis myer/add basic test structure for python 2 #2 --- .travis.yml | 5 +++-- Makefile | 7 +++++-- README.md | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2b65b12..dd26e0d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,8 +26,9 @@ before_install: - sudo apt-get update && sudo apt-get install elasticsearch -y - sudo service elasticsearch start -# command to install dependencies (e.g. tox) -install: pip install -r requirements/travis.txt +install: + - pip install -r requirements/travis.txt + - ./manage.py migrate --run-syncdb # sleep for elasticsearch before_script: diff --git a/Makefile b/Makefile index 412992f..d5ca9bc 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: clean coverage help \ - quality requirements selfcheck test test-all upgrade validate + quality requirements selfcheck syncdb test test-all upgrade validate .DEFAULT_GOAL := help @@ -52,7 +52,10 @@ coverage: clean ## check code coverage quickly with the default Python coverage html $(BROWSER) htmlcov/index.html -test: clean ## run tests in the current virtualenv +syncdb: clean ## setup local sqlite db + ./manage.py migrate --run-syncdb + +test: syncdb ## run tests in the current virtualenv ./manage.py test diff_cover: test diff --git a/README.md b/README.md index 3094d63..bdeced0 100644 --- a/README.md +++ b/README.md @@ -298,7 +298,7 @@ with `make upgrade`. ### Local Tests -Run `/.manage.py test --settings=bcore.settings +`make test` ### Updating Egg Info From 373820b623ed824cea4f1cbb24b9e1db4932614a Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Thu, 2 Aug 2018 07:52:39 -0400 Subject: [PATCH 16/24] move migrate to tox.ini from travis.yml myer/add basic test structure for python 2 #2 --- .travis.yml | 4 +--- tox.ini | 4 +++- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index dd26e0d..792bd7d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,9 +26,7 @@ before_install: - sudo apt-get update && sudo apt-get install elasticsearch -y - sudo service elasticsearch start -install: - - pip install -r requirements/travis.txt - - ./manage.py migrate --run-syncdb +install: pip install -r requirements/travis.txt # sleep for elasticsearch before_script: diff --git a/tox.ini b/tox.ini index c1520a0..d286e78 100644 --- a/tox.ini +++ b/tox.ini @@ -41,7 +41,9 @@ deps = es62: elasticsearch-dsl>=6.2.0,<6.3.0 -r{toxinidir}/requirements/test.txt -commands = ./manage.py test {posargs} +commands = + ./manage.py migrate --sync-db + ./manage.py test {posargs} [testenv:docs] setenv = From 1b8de2962900379d467d13e0c4a8e04fa7d01b45 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Thu, 2 Aug 2018 07:56:12 -0400 Subject: [PATCH 17/24] change to `./manage.py migrate --run-syncdb` in tox.ini myer/add basic test structure for python 2 #2 --- tox.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index d286e78..f1d30a8 100644 --- a/tox.ini +++ b/tox.ini @@ -42,7 +42,8 @@ deps = -r{toxinidir}/requirements/test.txt commands = - ./manage.py migrate --sync-db + ;https://docs.djangoproject.com/en/2.0/ref/django-admin/#cmdoption-migrate-run-syncdb + ./manage.py migrate --run-syncdb ./manage.py test {posargs} [testenv:docs] From 66f633613199c8850836ff0206508f10f5b6b7da Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Thu, 2 Aug 2018 12:55:17 -0400 Subject: [PATCH 18/24] add makefile `load` option myer/add basic test structure for python 2 #2 --- Makefile | 5 ++++- requirements/dev.txt | 2 +- requirements/travis.txt | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index d5ca9bc..565f6a6 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: clean coverage help \ - quality requirements selfcheck syncdb test test-all upgrade validate + quality requirements selfcheck syncdb load test test-all upgrade validate .DEFAULT_GOAL := help @@ -55,6 +55,9 @@ coverage: clean ## check code coverage quickly with the default Python syncdb: clean ## setup local sqlite db ./manage.py migrate --run-syncdb +load: syncdb ## load fixtures into sqlite + ./manage.py loaddata tests/tests_initial.json + test: syncdb ## run tests in the current virtualenv ./manage.py test diff --git a/requirements/dev.txt b/requirements/dev.txt index 1feaac7..d897810 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -13,7 +13,7 @@ click==6.7 # via pip-tools configparser==3.5.0 # via pydocstyle diff-cover==1.0.4 distlib==0.2.7 # via caniusepython3 -django==1.11.14 +django==1.11.15 elasticsearch-dsl==6.1.0 elasticsearch==6.3.0 # via elasticsearch-dsl first==2.0.1 # via pip-tools diff --git a/requirements/travis.txt b/requirements/travis.txt index eb6bdf3..750c5b7 100644 --- a/requirements/travis.txt +++ b/requirements/travis.txt @@ -18,4 +18,4 @@ six==1.11.0 # via packaging, tox tox-battery==0.5.1 tox==3.1.2 urllib3==1.23 # via requests -virtualenv==16.0.0 # via tox \ No newline at end of file +virtualenv==16.0.0 # via tox From 4342272fa8973be96cfe2a143d1760303ec2e32f Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Thu, 2 Aug 2018 15:58:22 -0400 Subject: [PATCH 19/24] add pylint and update docs. `make quality` is working. * remove `test_utils/__init__.py` * add pylint myer/add basic test structure for python 2 #2 --- Makefile | 1 - README.md | 10 +- pylintrc | 364 ++++++++++++++++++++++++++++++++++++--- pylintrc_tweaks | 4 + requirements/dev.in | 3 +- requirements/dev.txt | 20 ++- requirements/quality.in | 1 + requirements/quality.txt | 19 +- test_utils/__init__.py | 8 - tox.ini | 13 +- 10 files changed, 394 insertions(+), 49 deletions(-) create mode 100644 pylintrc_tweaks delete mode 100644 test_utils/__init__.py diff --git a/Makefile b/Makefile index 565f6a6..a51f90f 100644 --- a/Makefile +++ b/Makefile @@ -65,7 +65,6 @@ diff_cover: test diff-cover coverage.xml test-all: ## run tests on every supported Python/Django combination - # tox -e quality tox validate: quality test ## run tests and quality checks diff --git a/README.md b/README.md index bdeced0..8454301 100644 --- a/README.md +++ b/README.md @@ -296,9 +296,15 @@ This project also uses [`pip-tools`](https://github.com/jazzband/pip-tools). The `requirements.txt` files are generated and pinned to latest versions with `make upgrade`. -### Local Tests +### Running Tests Locally -`make test` +Run `make test`. To run all tests and quality checks locally, +run `make test-all`. + +To just run linting, `make quality`. Please note that if any of the +linters return a nonzero code, it will give an `InvocationError` error +at the end. See [tox's documentation for `InvocationError`](https://tox.readthedocs.io/en/latest/example/general.html#understanding-invocationerror-exit-codes) +for more information. ### Updating Egg Info diff --git a/pylintrc b/pylintrc index a21bfca..6a3ce4f 100644 --- a/pylintrc +++ b/pylintrc @@ -1,32 +1,350 @@ +# *************************** +# ** DO NOT EDIT THIS FILE ** +# *************************** +# +# This file was generated by edx-lint: http://github.com/edx/edx-lint +# +# If you want to change this file, you have two choices, depending on whether +# you want to make a local change that applies only to this repo, or whether +# you want to make a central change that applies to all repos using edx-lint. +# +# LOCAL CHANGE: +# +# 1. Edit the local pylintrc_tweaks file to add changes just to this +# repo's file. +# +# 2. Run: +# +# $ edx_lint write pylintrc +# +# 3. This will modify the local file. Submit a pull request to get it +# checked in so that others will benefit. +# +# +# CENTRAL CHANGE: +# +# 1. Edit the pylintrc file in the edx-lint repo at +# https://github.com/edx/edx-lint/blob/master/edx_lint/files/pylintrc +# +# 2. Make a new version of edx_lint, which involves the usual steps of +# incrementing the version number, submitting and reviewing a pull +# request, and updating the edx-lint version reference in this repo. +# +# 3. Install the newer version of edx-lint. +# +# 4. Run: +# +# $ edx_lint write pylintrc +# +# 5. This will modify the local file. Submit a pull request to get it +# checked in so that others will benefit. +# +# +# +# +# +# STAY AWAY FROM THIS FILE! +# +# +# +# +# +# SERIOUSLY. +# +# ------------------------------ [MASTER] ignore = migrations persistent = yes -load-plugins = caniusepython3.pylint_checker,pylint_django,pylint_celery +load-plugins = caniusepython3.pylint_checker,edx_lint.pylint,pylint_django [MESSAGES CONTROL] -disable = +enable = + # These are controlled by explicit choices in the pylintrc files + blacklisted-name, + line-too-long, + # These affect the correctness of the code + syntax-error, + init-is-generator, + return-in-init, + function-redefined, + not-in-loop, + return-outside-function, + yield-outside-function, + return-arg-in-generator, + nonexistent-operator, + duplicate-argument-name, + abstract-class-instantiated, + bad-reversed-sequence, + continue-in-finally, + method-hidden, + access-member-before-definition, + no-method-argument, + no-self-argument, + invalid-slots-object, + assigning-non-slot, + invalid-slots, + inherit-non-class, + inconsistent-mro, + duplicate-bases, + non-iterator-returned, + unexpected-special-method-signature, + invalid-length-returned, + import-error, + used-before-assignment, + undefined-variable, + undefined-all-variable, + invalid-all-object, + no-name-in-module, + unbalance-tuple-unpacking, + unpacking-non-sequence, + bad-except-order, + raising-bad-type, + misplaced-bare-raise, + raising-non-exception, + nonimplemented-raised, + catching-non-exception, + slots-on-old-class, + super-on-old-class, + bad-super-call, + missing-super-argument, + no-member, + not-callable, + assignment-from-no-return, + no-value-for-parameter, + too-many-function-args, + unexpected-keyword-arg, + redundant-keyword-arg, + invalid-sequence-index, + invalid-slice-index, + assignment-from-none, + not-context-manager, + invalid-unary-operand-type, + unsupported-binary-operation, + repeated-keyword, + not-an-iterable, + not-a-mapping, + unsupported-membership-test, + unsubscriptable-object, + logging-unsupported-format, + logging-too-many-args, + logging-too-few-args, + bad-format-character, + truncated-format-string, + mixed-fomat-string, + format-needs-mapping, + missing-format-string-key, + too-many-format-args, + too-few-format-args, + bad-str-strip-call, + model-unicode-not-callable, + super-method-not-called, + non-parent-method-called, + test-inherits-tests, + translation-of-non-string, + redefined-variable-type, + cyclical-import, + unreachable, + dangerous-default-value, + pointless-statement, + pointless-string-statement, + expression-not-assigned, + duplicate-key, + confusing-with-statement, + using-constant-test, + lost-exception, + assert-on-tuple, + attribute-defined-outside-init, + bad-staticmethod-argument, + arguments-differ, + signature-differs, + abstract-method, + super-init-not-called, + relative-import, + import-self, + misplaced-future, + invalid-encoded-data, + global-variable-undefined, + redefined-outer-name, + redefined-builtin, + redefined-in-handler, + undefined-loop-variable, + cell-var-from-loop, + duplicate-except, + nonstandard-exception, + binary-op-exception, + property-on-old-class, + bad-format-string-key, + unused-format-string-key, + bad-format-string, + missing-format-argument-key, + unused-format-string-argument, + format-combined-specification, + missing-format-attribute, + invalid-format-index, + anomalous-backslash-in-string, + anomalous-unicode-escape-in-string, + bad-open-mode, + boolean-datetime, + # Checking failed for some reason + fatal, + astroid-error, + parse-error, + method-check-failed, + django-not-available, + raw-checker-failed, + django-not-available-placeholder, + # Documentation is important + empty-docstring, + invalid-characters-in-docstring, + missing-docstring, + wrong-spelling-in-comment, + wrong-spelling-in-docstring, + # Unused code should be deleted + unused-import, + unused-variable, + unused-argument, + # These are dangerous! + exec-used, + eval-used, + # These represent idiomatic python. Not adhering to them + # will raise red flags with future readers. + bad-classmethod-argument, + bad-mcs-classmethod-argument, + bad-mcs-method-argument, + bad-whitespace, + consider-iterating-dictionary, + consider-using-enumerate, + literal-used-as-attribute, + multiple-imports, + multiple-statements, + old-style-class, + simplifiable-range, + singleton-comparison, + superfluous-parens, + unidiomatic-typecheck, + unneeded-not, + wrong-assert-type, + simplifiable-if-statement, + no-classmethod-decorator, + no-staticmethod-decorator, + unnecessary-pass, + unnecessary-lambda, + useless-else-on-loop, + unnecessary-semicolon, + reimported, + global-variable-not-assigned, + global-at-module-level, + bare-except, + broad-except, + logging-not-lazy, + redundant-unittest-assert, + model-missing-unicode, + model-has-unicode, + model-no-explicit-unicode, + protected-access, + # Don't use things that are deprecated + deprecated-module, + deprecated-method, + # These help manage code complexity + too-many-nested-blocks, + too-many-statements, + too-many-boolean-expressions, + # Consistent import order makes finding where code is + # imported from easier + ungrouped-imports, + wrong-import-order, + wrong-import-position, + wildcard-import, + # These should be auto-fixed by any competent editor + missing-final-newline, + mixed-line-endings, + trailing-newlines, + trailing-whitespace, + unexpected-line-ending-format, + mixed-indentation, + # These attempt to limit pylint line-noise + bad-option-value, + unrecognized-inline-option, + useless-suppression, + bad-inline-option, + deprecated-pragma, +disable = + # These should be left to the discretion of the reviewer + bad-continuation, + invalid-name, + misplaced-comparison-constant, + file-ignored, + bad-indentation, + lowercase-l-suffix, + unused-wildcard-import, + global-statement, + no-else-return, + # These are disabled by pylint by default + apply-builtin, + backtick, + basestring-builtin, + buffer-builtin, + cmp-builtin, + cmp-method, + coerce-builtin, + coerce-method, + delslice-method, + dict-iter-method, + dict-view-method, + duplicate-code, + execfile-builtin, + file-builtin, + filter-builtin-not-iterating, + fixme, + getslice-method, + hex-method, + import-star-module-level, + indexing-exception, + input-builtin, + intern-builtin, locally-disabled, locally-enabled, - too-few-public-methods, - bad-builtin, - star-args, - abstract-class-not-used, - abstract-class-little-used, - no-init, - fixme, logging-format-interpolation, - too-many-lines, + long-builtin, + long-suffix, + map-builtin-not-iterating, + metaclass-assignment, + next-method-called, + no-absolute-import, + no-init, no-self-use, + nonzero-method, + oct-method, + old-division, + old-ne-operator, + old-octal-literal, + old-raise-syntax, + parameter-unpacking, + print-statement, + raising-string, + range-builtin-not-iterating, + raw_input-builtin, + reduce-builtin, + reload-builtin, + round-builtin, + setslice-method, + standarderror-builtin, + suppressed-message, + too-few-public-methods, too-many-ancestors, + too-many-arguments, + too-many-branches, too-many-instance-attributes, - too-few-public-methods, + too-many-lines, + too-many-locals, too-many-public-methods, too-many-return-statements, - too-many-branches, - too-many-arguments, - too-many-locals, - unused-wildcard-import, - duplicate-code + unichr-builtin, + unicode-builtin, + unpacking-in-except, + using-cmp-argument, + xrange-builtin, + zip-builtin-not-iterating, [REPORTS] output-format = text @@ -49,7 +367,7 @@ inlinevar-rgx = [A-Za-z_][A-Za-z0-9_]*$ good-names = f,i,j,k,db,ex,Run,_,__ bad-names = foo,bar,baz,toto,tutu,tata no-docstring-rgx = __.*__$|test_.+|setUp$|setUpClass$|tearDown$|tearDownClass$|Meta$ -docstring-min-length = -1 +docstring-min-length = 5 [FORMAT] max-line-length = 120 @@ -72,7 +390,7 @@ ignore-imports = no ignore-mixin-members = yes ignored-classes = SQLObject unsafe-load-any-extension = yes -generated-members = +generated-members = REQUEST, acl_users, aq_parent, @@ -98,7 +416,7 @@ generated-members = [VARIABLES] init-import = no dummy-variables-rgx = _|dummy|unused|.*_unused -additional-builtins = +additional-builtins = [CLASSES] defining-attr-methods = __init__,__new__,setUp @@ -119,11 +437,11 @@ max-public-methods = 20 [IMPORTS] deprecated-modules = regsub,TERMIOS,Bastion,rexec -import-graph = -ext-import-graph = -int-import-graph = +import-graph = +ext-import-graph = +int-import-graph = [EXCEPTIONS] overgeneral-exceptions = Exception -# 1a67033d4799199101eddf63b8ed0bef3e61bda7 +# 78a6a959bb15570f4c637f393bac355d70724a5b diff --git a/pylintrc_tweaks b/pylintrc_tweaks new file mode 100644 index 0000000..eea29d6 --- /dev/null +++ b/pylintrc_tweaks @@ -0,0 +1,4 @@ +# pylintrc tweaks - put changes here, run `edx_lint write pylintrc` and it will generate pylintrc +[MASTER] +ignore = migrations +load-plugins = caniusepython3.pylint_checker,edx_lint.pylint,pylint_django \ No newline at end of file diff --git a/requirements/dev.in b/requirements/dev.in index af2d09f..ea485be 100644 --- a/requirements/dev.in +++ b/requirements/dev.in @@ -1,7 +1,8 @@ # Additional requirements for development of this application +edx-lint # includes pylint, pylint-django, etc; configured use pylintrc diff-cover # Changeset diff test coverage pip-tools # Requirements file management tox # virtualenv management for tests tox-battery # Makes tox aware of requirements file changes -wheel # For generation of wheels for PyPI +wheel # For generation of wheels for PyPI \ No newline at end of file diff --git a/requirements/dev.txt b/requirements/dev.txt index d897810..1697820 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -5,17 +5,21 @@ # pip-compile --output-file requirements/dev.txt requirements/base.in requirements/dev.in requirements/quality.in # argparse==1.4.0 # via caniusepython3 -backports.functools-lru-cache==1.5 # via caniusepython3 +astroid==1.5.2 # via edx-lint, pylint, pylint-celery +backports.functools-lru-cache==1.5 # via astroid, caniusepython3, pylint caniusepython3==7.0.0 certifi==2018.4.16 # via requests chardet==3.0.4 # via requests -click==6.7 # via pip-tools -configparser==3.5.0 # via pydocstyle +click-log==0.1.8 # via edx-lint +click==6.7 # via click-log, edx-lint, pip-tools +configparser==3.5.0 # via pydocstyle, pylint diff-cover==1.0.4 distlib==0.2.7 # via caniusepython3 django==1.11.15 +edx-lint==0.5.5 elasticsearch-dsl==6.1.0 elasticsearch==6.3.0 # via elasticsearch-dsl +enum34==1.1.6 # via astroid first==2.0.1 # via pip-tools futures==3.2.0 # via caniusepython3, isort idna==2.7 # via requests @@ -24,7 +28,9 @@ ipaddress==1.0.22 # via elasticsearch-dsl isort==4.3.4 jinja2-pluralize==0.3.0 # via diff-cover jinja2==2.10 # via diff-cover, jinja2-pluralize +lazy-object-proxy==1.3.1 # via astroid markupsafe==1.0 # via jinja2 +mccabe==0.6.1 # via pylint multiprocessing-logging==0.2.6 packaging==17.1 # via caniusepython3, tox pip-tools==2.0.2 @@ -33,11 +39,16 @@ py==1.5.4 # via tox pycodestyle==2.4.0 pydocstyle==2.1.1 pygments==2.2.0 # via diff-cover +pylint-celery==0.3 # via edx-lint +pylint-django==0.7.2 # via edx-lint +pylint-plugin-utils==0.4 # via pylint-celery, pylint-django +pylint==1.7.1 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils pyparsing==2.2.0 # via packaging python-dateutil==2.7.3 # via elasticsearch-dsl pytz==2018.5 # via django requests==2.19.1 # via caniusepython3 -six==1.11.0 # via diff-cover, elasticsearch-dsl, packaging, pip-tools, pydocstyle, python-dateutil, tox +singledispatch==3.4.0.3 # via astroid, pylint +six==1.11.0 # via astroid, diff-cover, edx-lint, elasticsearch-dsl, packaging, pip-tools, pydocstyle, pylint, python-dateutil, singledispatch, tox snowballstemmer==1.2.1 # via pydocstyle texttable==1.4.0 tox-battery==0.5.1 @@ -45,3 +56,4 @@ tox==3.1.2 urllib3==1.23 # via elasticsearch, requests virtualenv==16.0.0 # via tox wheel==0.31.1 +wrapt==1.10.11 # via astroid diff --git a/requirements/quality.in b/requirements/quality.in index 6e6fdc8..803c7e3 100644 --- a/requirements/quality.in +++ b/requirements/quality.in @@ -1,6 +1,7 @@ # Requirements for code quality checks caniusepython3 # Additional Python 3 compatibility pylint checks +edx-lint # includes pylint, pylint-django, etc; configured use pylintrc isort # to standardize order of imports pycodestyle # PEP 8 compliance validation pydocstyle # PEP 257 compliance validation diff --git a/requirements/quality.txt b/requirements/quality.txt index 55117fb..6aa44e2 100644 --- a/requirements/quality.txt +++ b/requirements/quality.txt @@ -5,20 +5,33 @@ # pip-compile --output-file requirements/quality.txt requirements/quality.in # argparse==1.4.0 # via caniusepython3 -backports.functools-lru-cache==1.5 # via caniusepython3 +astroid==1.5.2 # via edx-lint, pylint, pylint-celery +backports.functools-lru-cache==1.5 # via astroid, caniusepython3, pylint caniusepython3==7.0.0 certifi==2018.4.16 # via requests chardet==3.0.4 # via requests -configparser==3.5.0 # via pydocstyle +click-log==0.1.8 # via edx-lint +click==6.7 # via click-log, edx-lint +configparser==3.5.0 # via pydocstyle, pylint distlib==0.2.7 # via caniusepython3 +edx-lint==0.5.5 +enum34==1.1.6 # via astroid futures==3.2.0 # via caniusepython3, isort idna==2.7 # via requests isort==4.3.4 +lazy-object-proxy==1.3.1 # via astroid +mccabe==0.6.1 # via pylint packaging==17.1 # via caniusepython3 pycodestyle==2.4.0 pydocstyle==2.1.1 +pylint-celery==0.3 # via edx-lint +pylint-django==0.7.2 # via edx-lint +pylint-plugin-utils==0.4 # via pylint-celery, pylint-django +pylint==1.7.1 # via edx-lint, pylint-celery, pylint-django, pylint-plugin-utils pyparsing==2.2.0 # via packaging requests==2.19.1 # via caniusepython3 -six==1.11.0 # via packaging, pydocstyle +singledispatch==3.4.0.3 # via astroid, pylint +six==1.11.0 # via astroid, edx-lint, packaging, pydocstyle, pylint, singledispatch snowballstemmer==1.2.1 # via pydocstyle urllib3==1.23 # via requests +wrapt==1.10.11 # via astroid diff --git a/test_utils/__init__.py b/test_utils/__init__.py deleted file mode 100644 index d1fc6f5..0000000 --- a/test_utils/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -""" -Test utilities. - -Since py.test discourages putting __init__.py into test directory (i.e. making tests a package) -one cannot import from anywhere under tests folder. However, some utility classes/methods might be useful -in multiple test modules (i.e. factoryboy factories, base test classes). So this package is the place to put them. -""" -from __future__ import print_function diff --git a/tox.ini b/tox.ini index f1d30a8..d380c81 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] envlist = - {py27}-django19-{es60,es61} + {py27}-django19-{es61} ; not implemented yet - DocType Changes in es62 pending - see github issue #3 Support elasticsearch-dsl 6.2 ; {py27}-django19-{es62} @@ -64,19 +64,18 @@ commands = python setup.py check --restructuredtext --strict [testenv:quality] +setenv = + PYTHONIOENCODING=utf-8 whitelist_externals = make - rm touch deps = -r{toxinidir}/requirements/quality.txt -r{toxinidir}/requirements/test.txt commands = - touch tests/__init__.py - pylint django_elastic_migrations tests test_utils - pylint --py3k django_elastic_migrations tests test_utils - rm tests/__init__.py + pylint django_elastic_migrations tests + pylint --py3k django_elastic_migrations tests pycodestyle django_elastic_migrations tests pydocstyle django_elastic_migrations tests - isort --check-only --recursive tests test_utils django_elastic_migrations manage.py setup.py test_settings.py + isort --check-only --recursive tests django_elastic_migrations manage.py setup.py test_settings.py make selfcheck From a5e29e96b5396a1486a555dd0a50adad157aa07f Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Thu, 2 Aug 2018 16:08:19 -0400 Subject: [PATCH 20/24] welcome to the `__future__` myer/add basic test structure for python 2 #2 --- django_elastic_migrations/apps.py | 7 ++----- django_elastic_migrations/exceptions.py | 2 +- django_elastic_migrations/management/commands/es.py | 3 +-- .../management/commands/es_activate.py | 2 +- django_elastic_migrations/management/commands/es_clear.py | 2 +- django_elastic_migrations/management/commands/es_create.py | 2 +- .../management/commands/es_dangerous_reset.py | 2 +- .../management/commands/es_deactivate.py | 2 +- django_elastic_migrations/management/commands/es_drop.py | 2 +- django_elastic_migrations/management/commands/es_list.py | 2 +- django_elastic_migrations/management/commands/es_update.py | 2 +- django_elastic_migrations/utils/__init__.py | 2 +- .../utils/django_elastic_migrations_log.py | 2 +- django_elastic_migrations/utils/loading.py | 4 +--- django_elastic_migrations/utils/test_utils.py | 1 + tests/es_config.py | 2 +- tests/models.py | 2 +- tests/search.py | 1 + 18 files changed, 19 insertions(+), 23 deletions(-) diff --git a/django_elastic_migrations/apps.py b/django_elastic_migrations/apps.py index 7078cc0..7255a6c 100644 --- a/django_elastic_migrations/apps.py +++ b/django_elastic_migrations/apps.py @@ -1,13 +1,10 @@ # -*- coding: utf-8 -*- +from __future__ import (absolute_import, division, print_function, unicode_literals) + """ django_elastic_migrations Django application initialization. """ -from __future__ import print_function -from __future__ import absolute_import, unicode_literals - -import logging - from django.apps import AppConfig diff --git a/django_elastic_migrations/exceptions.py b/django_elastic_migrations/exceptions.py index e2cbdfc..4260fe4 100644 --- a/django_elastic_migrations/exceptions.py +++ b/django_elastic_migrations/exceptions.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django.core.management import CommandError diff --git a/django_elastic_migrations/management/commands/es.py b/django_elastic_migrations/management/commands/es.py index d8b0434..ac0a807 100644 --- a/django_elastic_migrations/management/commands/es.py +++ b/django_elastic_migrations/management/commands/es.py @@ -1,5 +1,4 @@ - -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django.core.management import BaseCommand, call_command, CommandError from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger diff --git a/django_elastic_migrations/management/commands/es_activate.py b/django_elastic_migrations/management/commands/es_activate.py index dee34eb..cb33db9 100644 --- a/django_elastic_migrations/management/commands/es_activate.py +++ b/django_elastic_migrations/management/commands/es_activate.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations import DEMIndexManager from django_elastic_migrations.management.commands.es import ESCommand diff --git a/django_elastic_migrations/management/commands/es_clear.py b/django_elastic_migrations/management/commands/es_clear.py index 6984169..6ea29f8 100644 --- a/django_elastic_migrations/management/commands/es_clear.py +++ b/django_elastic_migrations/management/commands/es_clear.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations import DEMIndexManager from django_elastic_migrations.management.commands.es import ESCommand from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger diff --git a/django_elastic_migrations/management/commands/es_create.py b/django_elastic_migrations/management/commands/es_create.py index 7a620ab..9a12109 100644 --- a/django_elastic_migrations/management/commands/es_create.py +++ b/django_elastic_migrations/management/commands/es_create.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations import DEMIndexManager from django_elastic_migrations.management.commands.es import ESCommand diff --git a/django_elastic_migrations/management/commands/es_dangerous_reset.py b/django_elastic_migrations/management/commands/es_dangerous_reset.py index 8469f96..5c64cd2 100644 --- a/django_elastic_migrations/management/commands/es_dangerous_reset.py +++ b/django_elastic_migrations/management/commands/es_dangerous_reset.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django.core.management import call_command from django_elastic_migrations import DEMIndexManager diff --git a/django_elastic_migrations/management/commands/es_deactivate.py b/django_elastic_migrations/management/commands/es_deactivate.py index 70bbab8..d48b52c 100644 --- a/django_elastic_migrations/management/commands/es_deactivate.py +++ b/django_elastic_migrations/management/commands/es_deactivate.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations import DEMIndexManager from django_elastic_migrations.management.commands.es import ESCommand diff --git a/django_elastic_migrations/management/commands/es_drop.py b/django_elastic_migrations/management/commands/es_drop.py index 90a50c3..cfe7d23 100644 --- a/django_elastic_migrations/management/commands/es_drop.py +++ b/django_elastic_migrations/management/commands/es_drop.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) import logging from django.core.management import CommandError diff --git a/django_elastic_migrations/management/commands/es_list.py b/django_elastic_migrations/management/commands/es_list.py index 071b897..13a38c2 100644 --- a/django_elastic_migrations/management/commands/es_list.py +++ b/django_elastic_migrations/management/commands/es_list.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from texttable import Texttable from django_elastic_migrations import DEMIndexManager diff --git a/django_elastic_migrations/management/commands/es_update.py b/django_elastic_migrations/management/commands/es_update.py index d762e4c..1c27075 100644 --- a/django_elastic_migrations/management/commands/es_update.py +++ b/django_elastic_migrations/management/commands/es_update.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from dateutil.parser import parse as dateutil_parse diff --git a/django_elastic_migrations/utils/__init__.py b/django_elastic_migrations/utils/__init__.py index 0170cf4..11b846a 100644 --- a/django_elastic_migrations/utils/__init__.py +++ b/django_elastic_migrations/utils/__init__.py @@ -1,2 +1,2 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) import importlib \ No newline at end of file diff --git a/django_elastic_migrations/utils/django_elastic_migrations_log.py b/django_elastic_migrations/utils/django_elastic_migrations_log.py index d092a86..c23b9e6 100644 --- a/django_elastic_migrations/utils/django_elastic_migrations_log.py +++ b/django_elastic_migrations/utils/django_elastic_migrations_log.py @@ -1,4 +1,4 @@ -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) import logging from multiprocessing_logging import install_mp_handler diff --git a/django_elastic_migrations/utils/loading.py b/django_elastic_migrations/utils/loading.py index e069587..b1b92d8 100644 --- a/django_elastic_migrations/utils/loading.py +++ b/django_elastic_migrations/utils/loading.py @@ -1,7 +1,5 @@ # encoding: utf-8 - -from __future__ import print_function -from __future__ import absolute_import, division, print_function, unicode_literals +from __future__ import (absolute_import, division, print_function, unicode_literals) from django_elastic_migrations.utils import importlib diff --git a/django_elastic_migrations/utils/test_utils.py b/django_elastic_migrations/utils/test_utils.py index 18b4f36..3e0bdbb 100644 --- a/django_elastic_migrations/utils/test_utils.py +++ b/django_elastic_migrations/utils/test_utils.py @@ -1,3 +1,4 @@ +from __future__ import (absolute_import, division, print_function, unicode_literals) from django.test import TestCase from django_elastic_migrations import DEMIndexManager diff --git a/tests/es_config.py b/tests/es_config.py index c5788fb..6044c19 100644 --- a/tests/es_config.py +++ b/tests/es_config.py @@ -1,5 +1,5 @@ # coding=utf-8 -from __future__ import print_function +from __future__ import (absolute_import, division, print_function, unicode_literals) from django.conf import settings from elasticsearch_dsl import connections from django.core.cache import caches diff --git a/tests/models.py b/tests/models.py index 6a1700a..e6647db 100644 --- a/tests/models.py +++ b/tests/models.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import unicode_literals +from __future__ import (absolute_import, division, print_function, unicode_literals) from django.db import models from django.utils.encoding import python_2_unicode_compatible diff --git a/tests/search.py b/tests/search.py index 62fdef1..f38268e 100644 --- a/tests/search.py +++ b/tests/search.py @@ -1,3 +1,4 @@ +from __future__ import (absolute_import, division, print_function, unicode_literals) from django.conf import settings from elasticsearch_dsl import Text, Q From 58d0103353c301697dd6a85009692b01b8039c9f Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Thu, 2 Aug 2018 16:52:07 -0400 Subject: [PATCH 21/24] update pylintrc myer/add basic test structure for python 2 #2 --- Makefile | 5 +- README.md | 3 + .../management/commands/es_drop.py | 3 - .../management/commands/es_list.py | 8 +- .../management/commands/es_update.py | 4 +- django_elastic_migrations/utils/__init__.py | 2 +- django_elastic_migrations/utils/es_utils.py | 8 +- pylintrc | 80 +------------------ pylintrc_tweaks | 5 +- tests/search.py | 2 +- 10 files changed, 25 insertions(+), 95 deletions(-) diff --git a/Makefile b/Makefile index a51f90f..99c3adc 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ .PHONY: clean coverage help \ - quality requirements selfcheck syncdb load test test-all upgrade validate + quality requirements selfcheck syncdb load test test-all upgrade validate pylintrc .DEFAULT_GOAL := help @@ -71,3 +71,6 @@ validate: quality test ## run tests and quality checks selfcheck: ## check that the Makefile is well-formed @echo "The Makefile is well-formed." + +pylintrc: ## check that the Makefile is well-formed + edx_lint write pylintrc diff --git a/README.md b/README.md index 8454301..6c71134 100644 --- a/README.md +++ b/README.md @@ -306,6 +306,9 @@ linters return a nonzero code, it will give an `InvocationError` error at the end. See [tox's documentation for `InvocationError`](https://tox.readthedocs.io/en/latest/example/general.html#understanding-invocationerror-exit-codes) for more information. +We use `edx_lint` to compile `pylintrc`. To update the rules, +change `pylintrc_tweaks` and run `make pylintrc`. + ### Updating Egg Info To update the `egg-info` directory, run `python setup.py egg_info` diff --git a/django_elastic_migrations/management/commands/es_drop.py b/django_elastic_migrations/management/commands/es_drop.py index cfe7d23..f762743 100644 --- a/django_elastic_migrations/management/commands/es_drop.py +++ b/django_elastic_migrations/management/commands/es_drop.py @@ -1,13 +1,10 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -import logging -from django.core.management import CommandError from django_elastic_migrations import DEMIndexManager from django_elastic_migrations.exceptions import CannotDropAllIndexesWithoutForceArg from django_elastic_migrations.management.commands.es import ESCommand from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger - logger = get_logger() diff --git a/django_elastic_migrations/management/commands/es_list.py b/django_elastic_migrations/management/commands/es_list.py index 13a38c2..073a866 100644 --- a/django_elastic_migrations/management/commands/es_list.py +++ b/django_elastic_migrations/management/commands/es_list.py @@ -7,7 +7,7 @@ from django_elastic_migrations.utils.django_elastic_migrations_log import get_logger -logger = get_logger() +log = get_logger() class Command(ESCommand): @@ -21,7 +21,7 @@ def add_arguments(self, parser): ) def handle(self, *args, **options): - logger.info("Available Index Definitions:") + log.info("Available Index Definitions:") indexes, _, apply_all, _, _ = self.get_index_specifying_options( options, require_one_include_list=['es_only']) @@ -80,8 +80,8 @@ def handle(self, *args, **options): except AttributeError: raise FirstMigrationNotRunError() - logger.info(table.draw()) - logger.info( + log.info(table.draw()) + log.info( "An index version name is: \n" "{environment prefix}{index name}-{version primary key id}. \n" "Most Django Elastic Migrations management commands take the \n" diff --git a/django_elastic_migrations/management/commands/es_update.py b/django_elastic_migrations/management/commands/es_update.py index 1c27075..69b6985 100644 --- a/django_elastic_migrations/management/commands/es_update.py +++ b/django_elastic_migrations/management/commands/es_update.py @@ -7,7 +7,7 @@ from django_elastic_migrations.utils.multiprocessing_utils import USE_ALL_WORKERS -logger = get_logger() +log = get_logger() class Command(ESCommand): @@ -60,7 +60,7 @@ def handle(self, *args, **options): if start_date is not None: if resume_mode: - logger.warning("--start takes precedence over --resume mode!") + log.warning("--start takes precedence over --resume mode!") start_date = dateutil_parse(start_date) if start_date: resume_mode = False diff --git a/django_elastic_migrations/utils/__init__.py b/django_elastic_migrations/utils/__init__.py index 11b846a..77b4367 100644 --- a/django_elastic_migrations/utils/__init__.py +++ b/django_elastic_migrations/utils/__init__.py @@ -1,2 +1,2 @@ from __future__ import (absolute_import, division, print_function, unicode_literals) -import importlib \ No newline at end of file +import importlib diff --git a/django_elastic_migrations/utils/es_utils.py b/django_elastic_migrations/utils/es_utils.py index d71b10d..334ee28 100644 --- a/django_elastic_migrations/utils/es_utils.py +++ b/django_elastic_migrations/utils/es_utils.py @@ -5,7 +5,7 @@ def get_index_hash_and_json(index): spec = index.to_dict() - json_str = json.dumps(spec) - hash = hashlib.md5() - hash.update(json_str) - return hash.hexdigest(), json_str + json_str = json.dumps(spec, sort_keys=True) + md5_hash = hashlib.md5() + md5_hash.update(json_str) + return md5_hash.hexdigest(), json_str diff --git a/pylintrc b/pylintrc index 6a3ce4f..ab70f89 100644 --- a/pylintrc +++ b/pylintrc @@ -268,83 +268,7 @@ enable = useless-suppression, bad-inline-option, deprecated-pragma, -disable = - # These should be left to the discretion of the reviewer - bad-continuation, - invalid-name, - misplaced-comparison-constant, - file-ignored, - bad-indentation, - lowercase-l-suffix, - unused-wildcard-import, - global-statement, - no-else-return, - # These are disabled by pylint by default - apply-builtin, - backtick, - basestring-builtin, - buffer-builtin, - cmp-builtin, - cmp-method, - coerce-builtin, - coerce-method, - delslice-method, - dict-iter-method, - dict-view-method, - duplicate-code, - execfile-builtin, - file-builtin, - filter-builtin-not-iterating, - fixme, - getslice-method, - hex-method, - import-star-module-level, - indexing-exception, - input-builtin, - intern-builtin, - locally-disabled, - locally-enabled, - logging-format-interpolation, - long-builtin, - long-suffix, - map-builtin-not-iterating, - metaclass-assignment, - next-method-called, - no-absolute-import, - no-init, - no-self-use, - nonzero-method, - oct-method, - old-division, - old-ne-operator, - old-octal-literal, - old-raise-syntax, - parameter-unpacking, - print-statement, - raising-string, - range-builtin-not-iterating, - raw_input-builtin, - reduce-builtin, - reload-builtin, - round-builtin, - setslice-method, - standarderror-builtin, - suppressed-message, - too-few-public-methods, - too-many-ancestors, - too-many-arguments, - too-many-branches, - too-many-instance-attributes, - too-many-lines, - too-many-locals, - too-many-public-methods, - too-many-return-statements, - unichr-builtin, - unicode-builtin, - unpacking-in-except, - using-cmp-argument, - xrange-builtin, - zip-builtin-not-iterating, +disable = C0111,unused-argument [REPORTS] output-format = text @@ -444,4 +368,4 @@ int-import-graph = [EXCEPTIONS] overgeneral-exceptions = Exception -# 78a6a959bb15570f4c637f393bac355d70724a5b +# 701d58a579f575603a2045f5e061520c889b54cf diff --git a/pylintrc_tweaks b/pylintrc_tweaks index eea29d6..848752a 100644 --- a/pylintrc_tweaks +++ b/pylintrc_tweaks @@ -1,4 +1,7 @@ # pylintrc tweaks - put changes here, run `edx_lint write pylintrc` and it will generate pylintrc [MASTER] ignore = migrations -load-plugins = caniusepython3.pylint_checker,edx_lint.pylint,pylint_django \ No newline at end of file +load-plugins = caniusepython3.pylint_checker,edx_lint.pylint,pylint_django + +[MESSAGES CONTROL] +disable=C0111,unused-argument diff --git a/tests/search.py b/tests/search.py index f38268e..d7c24ad 100644 --- a/tests/search.py +++ b/tests/search.py @@ -3,7 +3,7 @@ from elasticsearch_dsl import Text, Q from django_elastic_migrations.indexes import DEMIndex, DEMDocType -from models import Movie +from tests.models import Movie class GenericDocType(DEMDocType): From 807e30be8b4c5f36cde94104e2af9d79b8974441 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Fri, 3 Aug 2018 08:39:51 -0400 Subject: [PATCH 22/24] try the `allow_failures` option in `.travis.yml` myer/add basic test structure for python 2 #2 --- .travis.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 792bd7d..ed7f024 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,15 +7,17 @@ dist: trusty matrix: include: -# - env: TOX_ENV=py27-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt -# python: 2.7 + - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + python: 2.7 - env: TOX_ENV=py27-django19-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 2.7 -# TBD Python 3 tests ... -# - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt -# python: 3.6 -# - env: TOX_ENV=py36-django18-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt -# python: 3.6 + allow_failures: + - env: TOX_ENV=py27-django19-es62 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + python: 2.7 + - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + python: 3.6 + - env: TOX_ENV=py36-django18-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + python: 3.6 before_install: - pip install --upgrade pip From 963bd21ff97cf1ce675885c486b36abc06b1aae3 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Fri, 3 Aug 2018 09:04:11 -0400 Subject: [PATCH 23/24] comment out the `allow_failures` option in `.travis.yml` myer/add basic test structure for python 2 #2 --- .travis.yml | 15 ++++++++------- django_elastic_migrations/models.py | 2 +- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index ed7f024..a655721 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,13 +11,14 @@ matrix: python: 2.7 - env: TOX_ENV=py27-django19-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 2.7 - allow_failures: - - env: TOX_ENV=py27-django19-es62 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt - python: 2.7 - - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt - python: 3.6 - - env: TOX_ENV=py36-django18-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt - python: 3.6 +# TBD support - will be implemented in #5 +# allow_failures: +# - env: TOX_ENV=py27-django19-es62 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt +# python: 2.7 +# - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt +# python: 3.6 +# - env: TOX_ENV=py36-django18-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt +# python: 3.6 before_install: - pip install --upgrade pip diff --git a/django_elastic_migrations/models.py b/django_elastic_migrations/models.py index 305a66d..4eb39fd 100644 --- a/django_elastic_migrations/models.py +++ b/django_elastic_migrations/models.py @@ -1030,7 +1030,7 @@ def set_task_kwargs(self, kwargs): new_kwargs = {} for required_attribute in self.REQUIRED_TASK_KWARGS: new_kwargs[required_attribute] = kwargs[required_attribute] - self.task_kwargs = json.dumps(new_kwargs) + self.task_kwargs = json.dumps(new_kwargs, sort_keys=True) def perform_action(self, dem_index, *args, **kwargs): From f4443094b958115748b0ea0cbf4b2162f5e03ca6 Mon Sep 17 00:00:00 2001 From: Myer Nore Date: Fri, 3 Aug 2018 09:09:27 -0400 Subject: [PATCH 24/24] fix typo in `TOX_ENV`; py36 shouldn't match with python 2.7 myer/add basic test structure for python 2 #2 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a655721..c0cf0b5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ dist: trusty matrix: include: - - env: TOX_ENV=py36-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt + - env: TOX_ENV=py27-django19-es60 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 2.7 - env: TOX_ENV=py27-django19-es61 ES_APT_URL=https://artifacts.elastic.co/packages/6.x/apt python: 2.7