diff --git a/.travis.yml b/.travis.yml index 021fa5d7..31b04442 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,23 +1,31 @@ sudo: false language: python -python: -- '3.7' matrix: allow_failures: - env: TOXENV=py37-djangodev-sqlite - env: TOXENV=py37-djangodev-mysql - env: TOXENV=py37-djangodev-postgresql -env: - matrix: - - TOXENV=py27-django111-sqlite - - TOXENV=py37-django111-sqlite - - TOXENV=py37-django20-sqlite - - TOXENV=py37-django30-sqlite - - TOXENV=py37-djangodev-sqlite - - TOXENV=py37-djangodev-mysql - - TOXENV=py37-djangodev-postgresql - - TOXENV=checkqa - - TOXENV=docs + include: + - python: 3.7 + env: TOXENV=py37-django111-sqlite + - python: 3.7 + env: TOXENV=py37-django20-sqlite + - python: 3.7 + env: TOXENV=py37-django22-sqlite + - python: 3.7 + env: TOXENV=py37-django30-sqlite + - python: 3.8 + env: TOXENV=py38-django30-sqlite + - python: 3.7 + env: TOXENV=py37-djangodev-sqlite + - python: 3.7 + env: TOXENV=py37-djangodev-mysql + - python: 3.7 + env: TOXENV=py37-djangodev-postgresql + - python: 3.7 + env: TOXENV=checkqa + - python: 3.7 + env: TOXENV=docs install: - pip install -U pip - pip install tox codecov diff --git a/cities_light/abstract_models.py b/cities_light/abstract_models.py index 6bf52af0..3da9cc79 100644 --- a/cities_light/abstract_models.py +++ b/cities_light/abstract_models.py @@ -9,7 +9,7 @@ from django.db import models from django.db.models import lookups -from django.utils.encoding import force_text +from django.utils.encoding import force_str from django.conf import settings from django.utils.translation import ugettext_lazy as _ @@ -43,7 +43,7 @@ def to_ascii(value): For example, 'République Françaisen' would become 'Republique Francaisen' """ - return force_text(unidecode(value)) + return force_str(unidecode(value)) def to_search(value): @@ -148,6 +148,7 @@ class AbstractSubRegion(Base): country = models.ForeignKey(CITIES_LIGHT_APP_NAME + '.Country', on_delete=models.CASCADE) region = models.ForeignKey(CITIES_LIGHT_APP_NAME + '.Region', + null=True, blank=True, on_delete=models.CASCADE) class Meta(Base.Meta): @@ -198,7 +199,8 @@ class AbstractCity(Base): db_index=True, validators=[timezone_validator]) class Meta(Base.Meta): - unique_together = (('region', 'name'), ('region', 'slug')) + unique_together = (('region', 'subregion', 'name'), + ('region', 'subregion', 'slug')) verbose_name_plural = _('cities') abstract = True diff --git a/cities_light/admin.py b/cities_light/admin.py index 013963b1..81a1ee31 100644 --- a/cities_light/admin.py +++ b/cities_light/admin.py @@ -5,9 +5,8 @@ from django.contrib import admin from django.contrib.admin.views.main import ChangeList -from .forms import * -from .settings import * from .abstract_models import to_search +from . import forms from .loading import get_cities_models Country, Region, SubRegion, City = get_cities_models() @@ -38,7 +37,7 @@ class CountryAdmin(admin.ModelAdmin): list_filter = ( 'continent', ) - form = CountryForm + form = forms.CountryForm admin.site.register(Country, CountryAdmin) @@ -62,7 +61,7 @@ class RegionAdmin(admin.ModelAdmin): 'country', 'geoname_id', ) - form = RegionForm + form = forms.RegionForm admin.site.register(Region, RegionAdmin) @@ -72,6 +71,7 @@ class SubRegionAdmin(admin.ModelAdmin): """ ModelAdmin for SubRegion. """ + raw_id_fields = ["region"] list_filter = ( 'country__continent', 'country', @@ -88,24 +88,25 @@ class SubRegionAdmin(admin.ModelAdmin): 'region', 'geoname_id', ) - form = SubRegionForm + form = forms.SubRegionForm admin.site.register(SubRegion, SubRegionAdmin) class CityChangeList(ChangeList): - def get_query_set(self, request): + def get_queryset(self, request): if 'q' in list(request.GET.keys()): request.GET = copy(request.GET) request.GET['q'] = to_search(request.GET['q']) - return super(CityChangeList, self).get_query_set(request) + return super(CityChangeList, self).get_queryset(request) class CityAdmin(admin.ModelAdmin): """ ModelAdmin for City. """ + raw_id_fields = ["subregion", "region"] list_display = ( 'name', 'subregion', @@ -124,7 +125,7 @@ class CityAdmin(admin.ModelAdmin): 'country', 'timezone' ) - form = CityForm + form = forms.CityForm def get_changelist(self, request, **kwargs): return CityChangeList diff --git a/cities_light/downloader.py b/cities_light/downloader.py index 4aa0546e..cbf88d72 100644 --- a/cities_light/downloader.py +++ b/cities_light/downloader.py @@ -6,12 +6,9 @@ import time import os -try: - from urllib.request import urlopen - from urllib.parse import urlparse -except ImportError: - from urllib import urlopen - from urlparse import urlparse +from urllib.request import urlopen +from urllib.parse import urlparse + from .exceptions import SourceFileDoesNotExist diff --git a/cities_light/geonames.py b/cities_light/geonames.py index c2a29a29..b1978d36 100644 --- a/cities_light/geonames.py +++ b/cities_light/geonames.py @@ -1,6 +1,5 @@ from __future__ import unicode_literals -import six import os.path import zipfile import logging @@ -61,16 +60,10 @@ def extract(self, zip_path, file_name): zip_file.extract(file_name, DATA_DIR) def parse(self): - if not six.PY3: - file = open(self.file_path, 'r') - else: - file = open(self.file_path, encoding='utf-8', mode='r') + file = open(self.file_path, encoding='utf-8', mode='r') line = True for line in file: - if not six.PY3: - # In python3 this is already an unicode - line = line.decode('utf8') line = line.strip() # If the line is blank/empty or a comment, skip it and continue @@ -80,7 +73,4 @@ def parse(self): yield [e.strip() for e in line.split('\t')] def num_lines(self): - if not six.PY3: - return sum(1 for line in open(self.file_path)) - else: - return sum(1 for line in open(self.file_path, encoding='utf-8')) + return sum(1 for line in open(self.file_path, encoding='utf-8')) diff --git a/cities_light/loading.py b/cities_light/loading.py index 03ec83a2..007a0f88 100644 --- a/cities_light/loading.py +++ b/cities_light/loading.py @@ -1,12 +1,6 @@ -import django from .settings import CITIES_LIGHT_APP_NAME - - -if django.VERSION < (1, 7): - from django.db.models import get_model -else: - from django.apps import apps - get_model = apps.get_model +from django.apps import apps +get_model = apps.get_model def get_cities_model(model_name, *args, **kwargs): diff --git a/cities_light/management/commands/cities_light.py b/cities_light/management/commands/cities_light.py index 66418aba..6aa35373 100644 --- a/cities_light/management/commands/cities_light.py +++ b/cities_light/management/commands/cities_light.py @@ -7,6 +7,7 @@ import logging from argparse import RawTextHelpFormatter import sys + if sys.platform != 'win32': import resource @@ -71,34 +72,40 @@ def create_parser(self, *args, **kwargs): return parser def add_arguments(self, parser): - parser.add_argument('--force-import-all', action='store_true', + parser.add_argument( + '--force-import-all', action='store_true', default=False, help='Import even if files are up-to-date.' ), - parser.add_argument('--force-all', action='store_true', default=False, + parser.add_argument( + '--force-all', action='store_true', default=False, help='Download and import if files are up-to-date.' ), - parser.add_argument('--force-import', action='append', default=[], + parser.add_argument( + '--force-import', action='append', default=[], help='Import even if files matching files are up-to-date' ), - parser.add_argument('--force', action='append', default=[], + parser.add_argument( + '--force', action='append', default=[], help='Download and import even if matching files are up-to-date' ), parser.add_argument('--noinsert', action='store_true', - default=False, - help='Update existing data only' - ), - parser.add_argument('--hack-translations', action='store_true', + default=False, + help='Update existing data only' + ), + parser.add_argument( + '--hack-translations', action='store_true', default=False, help='Set this if you intend to import translations a lot' ), - parser.add_argument('--keep-slugs', action='store_true', + parser.add_argument( + '--keep-slugs', action='store_true', default=False, help='Do not update slugs' ), parser.add_argument('--progress', action='store_true', - default=False, - help='Show progress bar' - ), + default=False, + help='Show progress bar' + ), def progress_init(self): """Initialize progress bar.""" @@ -281,10 +288,12 @@ def _get_subregion_id(self, country_code2, region_id, subregion_id): if region_id not in self._region_codes[country_id]: self._region_codes[country_id][region_id] = Region.objects.get( country_id=country_id, geoname_code=region_id).pk + if subregion_id not in self._subregion_codes[country_id]: self._subregion_codes[country_id][subregion_id] = \ SubRegion.objects.get( - country_id=country_id, geoname_code=subregion_id).pk + region_id=self._region_codes[country_id][region_id], + geoname_code=subregion_id).pk return self._subregion_codes[country_id][subregion_id] def country_import(self, items): @@ -293,9 +302,10 @@ def country_import(self, items): except InvalidItems: return + force_insert = False + force_update = False try: - force_insert = False - force_update = False + country = Country.objects.get(geoname_id=items[ICountry.geonameid]) force_update = True except Country.DoesNotExist: @@ -336,9 +346,9 @@ def region_import(self, items): except InvalidItems: return + force_insert = False + force_update = False try: - force_insert = False - force_update = False region = Region.objects.get(geoname_id=items[IRegion.geonameid]) force_update = True except Region.DoesNotExist: @@ -393,8 +403,8 @@ def subregion_import(self, items): except InvalidItems: return + force_insert = force_update = False try: - force_insert = force_update = False subregion = SubRegion.objects.filter( geoname_id=items[ISubRegion.geonameid]).first() if subregion: @@ -471,9 +481,9 @@ def city_import(self, items): except InvalidItems: return + force_insert = False + force_update = False try: - force_insert = False - force_update = False city = City.objects.get(geoname_id=items[ICity.geonameid]) force_update = True except City.DoesNotExist: @@ -504,7 +514,7 @@ def city_import(self, items): items[ICity.admin1Code], items[ICity.admin2Code] ) - except SubRegion.DoesNotExist: + except (SubRegion.DoesNotExist, Region.DoesNotExist): subregion_id = None save = False diff --git a/cities_light/migrations/0010_auto_20200508_1851.py b/cities_light/migrations/0010_auto_20200508_1851.py new file mode 100644 index 00000000..3ef0ab7b --- /dev/null +++ b/cities_light/migrations/0010_auto_20200508_1851.py @@ -0,0 +1,24 @@ +# Generated by Django 3.0.5 on 2020-05-08 18:51 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('cities_light', '0009_add_subregion'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='city', + unique_together={('region', 'subregion', 'slug'), ('region', 'subregion', 'name')}, + ), + migrations.AlterField( + model_name='subregion', + name='region', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, + to='cities_light.Region'), + ), + ] diff --git a/cities_light/tests/base.py b/cities_light/tests/base.py index f097b62a..50e75063 100644 --- a/cities_light/tests/base.py +++ b/cities_light/tests/base.py @@ -43,7 +43,7 @@ class TestImportBase(test.TransactionTestCase): maxDiff = 100000 reset_sequences = True - def import_data(self, srcdir, countries, regions, subregions, cities, trans, **options): + def import_data(self, srcdir, countries, regions, subregions, cities, trans, file_type="txt", **options): """Helper method to import Geonames data. Patch *_SOURCES settings and call 'cities_light' command with @@ -58,18 +58,19 @@ def import_data(self, srcdir, countries, regions, subregions, cities, trans, **o trans - values for TRANSLATION_SOURCES **options - passed to call_command() as is """ + def _s2l(param): return param if isinstance(param, list) else [param] def _patch(setting, *values): setting_to_patch = ( - 'cities_light.management.commands.cities_light.%s_SOURCES' % - setting.upper() + 'cities_light.management.commands.cities_light.%s_SOURCES' % + setting.upper() ) return mock.patch( setting_to_patch, - ['file://%s.txt' % srcdir.get_file_path(v) for v in values] + ['file://%s.%s' % (srcdir.get_file_path(v), file_type) for v in values] ) m_country = _patch('country', *_s2l(countries)) @@ -78,6 +79,6 @@ def _patch(setting, *values): m_city = _patch('city', *_s2l(cities)) m_tr = _patch('translation', *_s2l(trans)) with m_country, m_region, m_subregion, m_city, m_tr: - management.call_command('cities_light', + management.call_command('cities_light', progress=True, force_import_all=True, **options) diff --git a/cities_light/tests/fixtures/import_zip/angouleme_city.zip b/cities_light/tests/fixtures/import_zip/angouleme_city.zip new file mode 100644 index 00000000..c7dae4bd Binary files /dev/null and b/cities_light/tests/fixtures/import_zip/angouleme_city.zip differ diff --git a/cities_light/tests/fixtures/import_zip/angouleme_country.zip b/cities_light/tests/fixtures/import_zip/angouleme_country.zip new file mode 100644 index 00000000..5a4b3385 Binary files /dev/null and b/cities_light/tests/fixtures/import_zip/angouleme_country.zip differ diff --git a/cities_light/tests/fixtures/import_zip/angouleme_region.zip b/cities_light/tests/fixtures/import_zip/angouleme_region.zip new file mode 100644 index 00000000..251113d2 Binary files /dev/null and b/cities_light/tests/fixtures/import_zip/angouleme_region.zip differ diff --git a/cities_light/tests/fixtures/import_zip/angouleme_subregion.zip b/cities_light/tests/fixtures/import_zip/angouleme_subregion.zip new file mode 100644 index 00000000..85582cdb Binary files /dev/null and b/cities_light/tests/fixtures/import_zip/angouleme_subregion.zip differ diff --git a/cities_light/tests/fixtures/import_zip/angouleme_translations.zip b/cities_light/tests/fixtures/import_zip/angouleme_translations.zip new file mode 100644 index 00000000..3c0fc01c Binary files /dev/null and b/cities_light/tests/fixtures/import_zip/angouleme_translations.zip differ diff --git a/cities_light/tests/test_import.py b/cities_light/tests/test_import.py index fb22b86b..ab620987 100644 --- a/cities_light/tests/test_import.py +++ b/cities_light/tests/test_import.py @@ -1,7 +1,11 @@ from __future__ import unicode_literals +import glob +import os + from dbdiff.fixture import Fixture from .base import TestImportBase, FixtureDir +from ..settings import DATA_DIR class TestImport(TestImportBase): @@ -20,6 +24,24 @@ def test_single_city(self): ) Fixture(fixture_dir.get_file_path('angouleme.json')).assertNoDiff() + def test_single_city_zip(self): + """Load single city.""" + filelist = glob.glob(os.path.join(DATA_DIR, "angouleme_*.txt")) + for f in filelist: + os.remove(f) + + fixture_dir = FixtureDir('import_zip') + self.import_data( + fixture_dir, + 'angouleme_country', + 'angouleme_region', + 'angouleme_subregion', + 'angouleme_city', + 'angouleme_translations', + file_type="zip" + ) + Fixture(FixtureDir('import').get_file_path('angouleme.json')).assertNoDiff() + def test_city_wrong_timezone(self): """Load single city with wrong timezone.""" fixture_dir = FixtureDir('import') diff --git a/cities_light/validators.py b/cities_light/validators.py index e4c7738c..09d0f58e 100644 --- a/cities_light/validators.py +++ b/cities_light/validators.py @@ -3,7 +3,7 @@ import pytz from django.core.exceptions import ValidationError -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ def timezone_validator(value): diff --git a/test_project/settings.py b/test_project/settings.py index c383bab6..bf960d1e 100644 --- a/test_project/settings.py +++ b/test_project/settings.py @@ -16,7 +16,6 @@ # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - # Quick-start development settings - unsuitable for production # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ @@ -28,7 +27,6 @@ ALLOWED_HOSTS = [] - # Application definition INSTALLED_APPS = [ @@ -42,15 +40,15 @@ ] # Rename to MIDDLEWARE on Django 1.10 -MIDDLEWARE_CLASSES = [ - # 'django.middleware.security.SecurityMiddleware', +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.locale.LocaleMiddleware', ] ROOT_URLCONF = 'urls' @@ -73,7 +71,6 @@ WSGI_APPLICATION = 'wsgi.application' - # Database # https://docs.djangoproject.com/en/1.10/ref/settings/#databases @@ -108,7 +105,6 @@ }, ] - # Internationalization # https://docs.djangoproject.com/en/1.10/topics/i18n/ @@ -122,7 +118,6 @@ USE_TZ = True - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.10/howto/static-files/ @@ -147,27 +142,27 @@ 'filters': ['require_debug_false'], 'class': 'django.utils.log.AdminEmailHandler' }, - 'console':{ - 'level':'DEBUG', - 'class':'logging.StreamHandler', + 'console': { + 'level': 'DEBUG', + 'class': 'logging.StreamHandler', }, }, 'loggers': { 'django.request': { - 'handlers':['console'], + 'handlers': ['console'], 'propagate': True, - 'level':'DEBUG', + 'level': 'DEBUG', }, 'cities_light': { - 'handlers':['console'], + 'handlers': ['console'], 'propagate': True, - 'level':'WARNING', + 'level': 'INFO', }, } } if os.environ.get('CI', False): - CITIES_LIGHT_TRANSLATION_LANGUAGES=['fr', 'ru'] + CITIES_LIGHT_TRANSLATION_LANGUAGES = ['fr', 'ru'] FIXTURE_DIR = os.path.abspath( os.path.join(BASE_DIR, 'cities_light', 'tests', 'fixtures') diff --git a/test_project/tests.py b/test_project/tests.py index 58a7cfec..a4994c0a 100644 --- a/test_project/tests.py +++ b/test_project/tests.py @@ -1,7 +1,8 @@ # -*- encoding: utf-8 -*- from __future__ import unicode_literals -from django.utils import unittest +from django.contrib.auth import get_user_model +from django.test import TestCase from django.test.client import RequestFactory from django.db.models import query from django.contrib.admin.sites import AdminSite @@ -10,19 +11,21 @@ from cities_light import models as cl_models -class AdminTestCase(unittest.TestCase): +class AdminTestCase(TestCase): def setUp(self): self.factory = RequestFactory() self.admin_site = AdminSite() def testCityChangeList(self): + user = get_user_model().objects.create(is_superuser=True, username="test") request = self.factory.get('/some/path/', data={'q': 'some query'}) + request.user = user city_admin = cl_admin.CityAdmin(cl_models.City, self.admin_site) changelist = cl_admin.CityChangeList( request, cl_models.City, cl_admin.CityAdmin.list_display, cl_admin.CityAdmin.list_display_links, cl_admin.CityAdmin.list_filter, cl_admin.CityAdmin.date_hierarchy, cl_admin.CityAdmin.search_fields, cl_admin.CityAdmin.list_select_related, cl_admin.CityAdmin.list_per_page, - cl_admin.CityAdmin.list_max_show_all, cl_admin.CityAdmin.list_editable, city_admin) + cl_admin.CityAdmin.list_max_show_all, cl_admin.CityAdmin.list_editable, city_admin, "id") - self.assertIsInstance(changelist.get_query_set(request), query.QuerySet) + self.assertIsInstance(changelist.get_queryset(request), query.QuerySet) diff --git a/test_project/urls.py b/test_project/urls.py index 388fcde4..0d682f62 100644 --- a/test_project/urls.py +++ b/test_project/urls.py @@ -1,8 +1,6 @@ -import django - -from django.conf.urls import include, url +from django.conf.urls import url from django.contrib import admin urlpatterns = [ - url(r'^admin/', include(admin.site.urls)), + url(r'^admin/', admin.site.urls), ] diff --git a/tox.ini b/tox.ini index a2a47c6d..f1399973 100644 --- a/tox.ini +++ b/tox.ini @@ -1,7 +1,7 @@ [tox] envlist = - py{27,35,36}-django{111,21,30}{-sqlite,-mysql,-postgresql}, - py{35,36}-djangodev{-sqlite,-mysql,-postgresql}, + py{35,36, 37,38}-django{111,22,30}{-sqlite,-mysql,-postgresql}, + py{35,36, 37,38}-djangodev{-sqlite,-mysql,-postgresql}, checkqa, docs skip_missing_interpreters = True