Merge pull request #220 from jazzband/development

Development

.travis.yml

@@ -1,23 +1,31 @@
 sudo: false
 language: python
-python:
-- '3.7'
 matrix:
   allow_failures:
   - env: TOXENV=py37-djangodev-sqlite
   - env: TOXENV=py37-djangodev-mysql
   - env: TOXENV=py37-djangodev-postgresql
-env:
-  matrix:
-  - TOXENV=py27-django111-sqlite
-  - TOXENV=py37-django111-sqlite
-  - TOXENV=py37-django20-sqlite
-  - TOXENV=py37-django30-sqlite
-  - TOXENV=py37-djangodev-sqlite
-  - TOXENV=py37-djangodev-mysql
-  - TOXENV=py37-djangodev-postgresql
-  - TOXENV=checkqa
-  - TOXENV=docs
+  include:
+  - python: 3.7
+    env: TOXENV=py37-django111-sqlite
+  - python: 3.7
+    env: TOXENV=py37-django20-sqlite
+  - python: 3.7
+    env: TOXENV=py37-django22-sqlite
+  - python: 3.7
+    env: TOXENV=py37-django30-sqlite
+  - python: 3.8
+    env: TOXENV=py38-django30-sqlite
+  - python: 3.7
+    env: TOXENV=py37-djangodev-sqlite
+  - python: 3.7
+    env: TOXENV=py37-djangodev-mysql
+  - python: 3.7
+    env: TOXENV=py37-djangodev-postgresql
+  - python: 3.7
+    env: TOXENV=checkqa
+  - python: 3.7
+    env: TOXENV=docs
 install:
 - pip install -U pip
 - pip install tox codecov

cities_light/abstract_models.py

@@ -9,7 +9,7 @@
 
 from django.db import models
 from django.db.models import lookups
-from django.utils.encoding import force_text
+from django.utils.encoding import force_str
 from django.conf import settings
 from django.utils.translation import ugettext_lazy as _
 
@@ -43,7 +43,7 @@ def to_ascii(value):
 
     For example, 'République Françaisen' would become 'Republique Francaisen'
     """
-    return force_text(unidecode(value))
+    return force_str(unidecode(value))
 
 
 def to_search(value):
@@ -148,6 +148,7 @@ class AbstractSubRegion(Base):
     country = models.ForeignKey(CITIES_LIGHT_APP_NAME + '.Country',
                                 on_delete=models.CASCADE)
     region = models.ForeignKey(CITIES_LIGHT_APP_NAME + '.Region',
+                               null=True, blank=True,
                                on_delete=models.CASCADE)
 
     class Meta(Base.Meta):
@@ -198,7 +199,8 @@ class AbstractCity(Base):
                                 db_index=True, validators=[timezone_validator])
 
     class Meta(Base.Meta):
-        unique_together = (('region', 'name'), ('region', 'slug'))
+        unique_together = (('region', 'subregion', 'name'),
+                           ('region', 'subregion', 'slug'))
         verbose_name_plural = _('cities')
         abstract = True
 

cities_light/admin.py

@@ -5,9 +5,8 @@
 from django.contrib import admin
 from django.contrib.admin.views.main import ChangeList
 
-from .forms import *
-from .settings import *
 from .abstract_models import to_search
+from . import forms
 from .loading import get_cities_models
 
 Country, Region, SubRegion, City = get_cities_models()
@@ -38,7 +37,7 @@ class CountryAdmin(admin.ModelAdmin):
     list_filter = (
         'continent',
     )
-    form = CountryForm
+    form = forms.CountryForm
 
 
 admin.site.register(Country, CountryAdmin)
@@ -62,7 +61,7 @@ class RegionAdmin(admin.ModelAdmin):
         'country',
         'geoname_id',
     )
-    form = RegionForm
+    form = forms.RegionForm
 
 
 admin.site.register(Region, RegionAdmin)
@@ -72,6 +71,7 @@ class SubRegionAdmin(admin.ModelAdmin):
     """
     ModelAdmin for SubRegion.
     """
+    raw_id_fields = ["region"]
     list_filter = (
         'country__continent',
         'country',
@@ -88,24 +88,25 @@ class SubRegionAdmin(admin.ModelAdmin):
         'region',
         'geoname_id',
     )
-    form = SubRegionForm
+    form = forms.SubRegionForm
 
 
 admin.site.register(SubRegion, SubRegionAdmin)
 
 
 class CityChangeList(ChangeList):
-    def get_query_set(self, request):
+    def get_queryset(self, request):
         if 'q' in list(request.GET.keys()):
             request.GET = copy(request.GET)
             request.GET['q'] = to_search(request.GET['q'])
-        return super(CityChangeList, self).get_query_set(request)
+        return super(CityChangeList, self).get_queryset(request)
 
 
 class CityAdmin(admin.ModelAdmin):
     """
     ModelAdmin for City.
     """
+    raw_id_fields = ["subregion", "region"]
     list_display = (
         'name',
         'subregion',
@@ -124,7 +125,7 @@ class CityAdmin(admin.ModelAdmin):
         'country',
         'timezone'
     )
-    form = CityForm
+    form = forms.CityForm
 
     def get_changelist(self, request, **kwargs):
         return CityChangeList

cities_light/downloader.py

@@ -6,12 +6,9 @@
 import time
 import os
 
-try:
-    from urllib.request import urlopen
-    from urllib.parse import urlparse
-except ImportError:
-    from urllib import urlopen
-    from urlparse import urlparse
+from urllib.request import urlopen
+from urllib.parse import urlparse
+
 
 from .exceptions import SourceFileDoesNotExist
 

cities_light/geonames.py

@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import six
 import os.path
 import zipfile
 import logging
@@ -61,16 +60,10 @@ def extract(self, zip_path, file_name):
             zip_file.extract(file_name, DATA_DIR)
 
     def parse(self):
-        if not six.PY3:
-            file = open(self.file_path, 'r')
-        else:
-            file = open(self.file_path, encoding='utf-8', mode='r')
+        file = open(self.file_path, encoding='utf-8', mode='r')
         line = True
 
         for line in file:
-            if not six.PY3:
-                # In python3 this is already an unicode
-                line = line.decode('utf8')
 
             line = line.strip()
             # If the line is blank/empty or a comment, skip it and continue
@@ -80,7 +73,4 @@ def parse(self):
             yield [e.strip() for e in line.split('\t')]
 
     def num_lines(self):
-        if not six.PY3:
-            return sum(1 for line in open(self.file_path))
-        else:
-            return sum(1 for line in open(self.file_path, encoding='utf-8'))
+        return sum(1 for line in open(self.file_path, encoding='utf-8'))

cities_light/loading.py

@@ -1,12 +1,6 @@
-import django
 from .settings import CITIES_LIGHT_APP_NAME
-
-
-if django.VERSION < (1, 7):
-    from django.db.models import get_model
-else:
-    from django.apps import apps
-    get_model = apps.get_model
+from django.apps import apps
+get_model = apps.get_model
 
 
 def get_cities_model(model_name, *args, **kwargs):

cities_light/management/commands/cities_light.py

@@ -7,6 +7,7 @@
 import logging
 from argparse import RawTextHelpFormatter
 import sys
+
 if sys.platform != 'win32':
     import resource
 
@@ -71,34 +72,40 @@ def create_parser(self, *args, **kwargs):
         return parser
 
     def add_arguments(self, parser):
-        parser.add_argument('--force-import-all', action='store_true',
+        parser.add_argument(
+            '--force-import-all', action='store_true',
             default=False, help='Import even if files are up-to-date.'
         ),
-        parser.add_argument('--force-all', action='store_true', default=False,
+        parser.add_argument(
+            '--force-all', action='store_true', default=False,
             help='Download and import if files are up-to-date.'
         ),
-        parser.add_argument('--force-import', action='append', default=[],
+        parser.add_argument(
+            '--force-import', action='append', default=[],
             help='Import even if files matching files are up-to-date'
         ),
-        parser.add_argument('--force', action='append', default=[],
+        parser.add_argument(
+            '--force', action='append', default=[],
             help='Download and import even if matching files are up-to-date'
         ),
         parser.add_argument('--noinsert', action='store_true',
-            default=False,
-            help='Update existing data only'
-        ),
-        parser.add_argument('--hack-translations', action='store_true',
+                            default=False,
+                            help='Update existing data only'
+                            ),
+        parser.add_argument(
+            '--hack-translations', action='store_true',
             default=False,
             help='Set this if you intend to import translations a lot'
         ),
-        parser.add_argument('--keep-slugs', action='store_true',
+        parser.add_argument(
+            '--keep-slugs', action='store_true',
             default=False,
             help='Do not update slugs'
         ),
         parser.add_argument('--progress', action='store_true',
-            default=False,
-            help='Show progress bar'
-        ),
+                            default=False,
+                            help='Show progress bar'
+                            ),
 
     def progress_init(self):
         """Initialize progress bar."""
@@ -281,10 +288,12 @@ def _get_subregion_id(self, country_code2, region_id, subregion_id):
         if region_id not in self._region_codes[country_id]:
             self._region_codes[country_id][region_id] = Region.objects.get(
                 country_id=country_id, geoname_code=region_id).pk
+
         if subregion_id not in self._subregion_codes[country_id]:
             self._subregion_codes[country_id][subregion_id] = \
                 SubRegion.objects.get(
-                country_id=country_id, geoname_code=subregion_id).pk
+                    region_id=self._region_codes[country_id][region_id],
+                    geoname_code=subregion_id).pk
         return self._subregion_codes[country_id][subregion_id]
 
     def country_import(self, items):
@@ -293,9 +302,10 @@ def country_import(self, items):
         except InvalidItems:
             return
 
+        force_insert = False
+        force_update = False
         try:
-            force_insert = False
-            force_update = False
+
             country = Country.objects.get(geoname_id=items[ICountry.geonameid])
             force_update = True
         except Country.DoesNotExist:
@@ -336,9 +346,9 @@ def region_import(self, items):
         except InvalidItems:
             return
 
+        force_insert = False
+        force_update = False
         try:
-            force_insert = False
-            force_update = False
             region = Region.objects.get(geoname_id=items[IRegion.geonameid])
             force_update = True
         except Region.DoesNotExist:
@@ -393,8 +403,8 @@ def subregion_import(self, items):
         except InvalidItems:
             return
 
+        force_insert = force_update = False
         try:
-            force_insert = force_update = False
             subregion = SubRegion.objects.filter(
                 geoname_id=items[ISubRegion.geonameid]).first()
             if subregion:
@@ -471,9 +481,9 @@ def city_import(self, items):
         except InvalidItems:
             return
 
+        force_insert = False
+        force_update = False
         try:
-            force_insert = False
-            force_update = False
             city = City.objects.get(geoname_id=items[ICity.geonameid])
             force_update = True
         except City.DoesNotExist:
@@ -504,7 +514,7 @@ def city_import(self, items):
                 items[ICity.admin1Code],
                 items[ICity.admin2Code]
             )
-        except SubRegion.DoesNotExist:
+        except (SubRegion.DoesNotExist, Region.DoesNotExist):
             subregion_id = None
 
         save = False

cities_light/migrations/0010_auto_20200508_1851.py

@@ -0,0 +1,24 @@
+# Generated by Django 3.0.5 on 2020-05-08 18:51
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('cities_light', '0009_add_subregion'),
+    ]
+
+    operations = [
+        migrations.AlterUniqueTogether(
+            name='city',
+            unique_together={('region', 'subregion', 'slug'), ('region', 'subregion', 'name')},
+        ),
+        migrations.AlterField(
+            model_name='subregion',
+            name='region',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE,
+                                    to='cities_light.Region'),
+        ),
+    ]