From 4ca942274d047fcd00bc8e946ec2bc383fa6cd54 Mon Sep 17 00:00:00 2001 From: james-otten Date: Sun, 31 Mar 2024 12:53:56 -0400 Subject: [PATCH] Linting fixes (#272) * isort fixes via `isort .` * black fixes via `black .` * enable isort in `invoke lint` * Add isort to `invoke format` * resolve various flake8 findings * format commented out code I should have removed to appease black * nuke large commented out code block --------- Co-authored-by: Andrew Dickinson --- src/meshapi/admin.py | 2 +- src/meshapi/docs.py | 4 +- .../management/commands/scramble_members.py | 6 +- src/meshapi/migrations/0001_initial.py | 2 +- ...all_options_building_panoramas_and_more.py | 2 +- src/meshapi/tests/group_helpers.py | 2 +- src/meshapi/tests/test_admin_change_view.py | 2 +- src/meshapi/tests/test_device.py | 2 +- .../tests/test_foreign_key_views_and_perms.py | 4 +- src/meshapi/tests/test_join_form.py | 2 +- src/meshapi/tests/test_kml_endpoint.py | 6 +- src/meshapi/tests/test_link.py | 2 +- src/meshapi/tests/test_lookups.py | 2 +- src/meshapi/tests/test_map_endpoints.py | 4 +- src/meshapi/tests/test_node.py | 4 +- src/meshapi/tests/test_sector.py | 2 +- src/meshapi/tests/test_update_panos_github.py | 2 +- src/meshapi/tests/test_views_post_delete.py | 12 +-- src/meshapi/urls.py | 3 +- src/meshapi/util/django_pglocks.py | 2 +- src/meshapi/validation.py | 15 ++- src/meshapi/views/__init__.py | 2 +- src/meshapi/views/forms.py | 6 +- src/meshapi/views/geography.py | 2 +- src/meshapi/views/map.py | 2 +- src/meshapi/views/model_api.py | 2 - src/meshapi/views/panoramas.py | 9 +- src/meshapi/views/query_api.py | 20 +--- src/meshapi_hooks/migrations/0001_initial.py | 4 +- .../0002_celeryserializerhook_and_more.py | 4 +- src/meshapi_hooks/tests/test_webhooks.py | 6 +- src/meshdb/urls.py | 3 +- .../building/resolve_address.py | 91 +++---------------- .../utils/spreadsheet_import/csv_load.py | 2 +- src/meshdb/utils/spreadsheet_import/main.py | 4 +- .../spreadsheet_import/parse_building.py | 10 +- .../utils/spreadsheet_import/parse_devices.py | 9 +- .../utils/spreadsheet_import/parse_link.py | 3 +- .../tests/test_humanify_addr.py | 2 - src/meshweb/tests.py | 2 +- src/meshweb/urls.py | 3 +- tasks.py | 6 +- 42 files changed, 91 insertions(+), 183 deletions(-) diff --git a/src/meshapi/admin.py b/src/meshapi/admin.py index d13a0c22..262ee0de 100644 --- a/src/meshapi/admin.py +++ b/src/meshapi/admin.py @@ -1,6 +1,6 @@ from django.contrib import admin from django.contrib.admin.options import forms -from django.db.models import Q, QuerySet +from django.db.models import Q from django.utils.safestring import mark_safe from nonrelated_inlines.admin import NonrelatedTabularInline diff --git a/src/meshapi/docs.py b/src/meshapi/docs.py index 47f8a1d1..39c81821 100644 --- a/src/meshapi/docs.py +++ b/src/meshapi/docs.py @@ -29,8 +29,8 @@ def get(self, request, *args, **kwargs): deepLinking: true, onComplete: ui_onComplete, plugins: [LogoutViaDjango], - }}; - const logoutURL = "{reverse("rest_framework:logout")}"; + }}; + const logoutURL = "{reverse("rest_framework:logout")}"; """ if session_id: response.data["settings"] += 'const sessionID = "{session_id}";' diff --git a/src/meshapi/management/commands/scramble_members.py b/src/meshapi/management/commands/scramble_members.py index b23b9d2c..2a250abc 100644 --- a/src/meshapi/management/commands/scramble_members.py +++ b/src/meshapi/management/commands/scramble_members.py @@ -1,9 +1,9 @@ -from django.contrib.auth.models import Group, Permission +from random import randrange + from django.core.management.base import BaseCommand from faker import Faker -from random import randrange -from meshapi.models import Member, Install +from meshapi.models import Install, Member # Uses faker to get fake names, emails, and phone numbers diff --git a/src/meshapi/migrations/0001_initial.py b/src/meshapi/migrations/0001_initial.py index 0cac928b..4db78d83 100644 --- a/src/meshapi/migrations/0001_initial.py +++ b/src/meshapi/migrations/0001_initial.py @@ -2,8 +2,8 @@ import django.contrib.postgres.fields import django.core.validators -from django.db import migrations, models import django.db.models.deletion +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/src/meshapi/migrations/0003_alter_install_options_building_panoramas_and_more.py b/src/meshapi/migrations/0003_alter_install_options_building_panoramas_and_more.py index 2ebdf534..2c4f410b 100644 --- a/src/meshapi/migrations/0003_alter_install_options_building_panoramas_and_more.py +++ b/src/meshapi/migrations/0003_alter_install_options_building_panoramas_and_more.py @@ -1,7 +1,7 @@ # Generated by Django 4.2.11 on 2024-03-22 01:45 -from django.db import migrations, models import django_jsonform.models.fields +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/src/meshapi/tests/group_helpers.py b/src/meshapi/tests/group_helpers.py index f7694c36..c921fe85 100644 --- a/src/meshapi/tests/group_helpers.py +++ b/src/meshapi/tests/group_helpers.py @@ -1,4 +1,4 @@ -from django.contrib.auth.models import Group, Permission +from django.contrib.auth.models import Group from django.core.management import call_command diff --git a/src/meshapi/tests/test_admin_change_view.py b/src/meshapi/tests/test_admin_change_view.py index 5450620f..154a212f 100644 --- a/src/meshapi/tests/test_admin_change_view.py +++ b/src/meshapi/tests/test_admin_change_view.py @@ -1,7 +1,7 @@ from django.contrib.auth.models import User from django.test import Client, TestCase -from meshapi.models import Building, Device, Install, Link, Member, Node, Sector +from meshapi.models import Building, Device, Install, Link, Member, Node from .sample_data import sample_building, sample_device, sample_install, sample_member, sample_node diff --git a/src/meshapi/tests/test_device.py b/src/meshapi/tests/test_device.py index 5957c752..70933130 100644 --- a/src/meshapi/tests/test_device.py +++ b/src/meshapi/tests/test_device.py @@ -3,7 +3,7 @@ from django.contrib.auth.models import User from django.test import Client, TestCase -from ..models import Building, Device, Link, Node +from ..models import Device, Node from .sample_data import sample_device, sample_node diff --git a/src/meshapi/tests/test_foreign_key_views_and_perms.py b/src/meshapi/tests/test_foreign_key_views_and_perms.py index d7e6c42e..1fe62284 100644 --- a/src/meshapi/tests/test_foreign_key_views_and_perms.py +++ b/src/meshapi/tests/test_foreign_key_views_and_perms.py @@ -3,8 +3,8 @@ from django.contrib.auth.models import Permission, User from django.test import Client, TestCase -from meshapi.models import Building, Install, Link, Member, Node, Sector -from meshapi.tests.sample_data import sample_building, sample_install, sample_member, sample_node +from meshapi.models import Building, Install, Member +from meshapi.tests.sample_data import sample_building, sample_install, sample_member def setup_objects(): diff --git a/src/meshapi/tests/test_join_form.py b/src/meshapi/tests/test_join_form.py index eaadfd1c..7c9aa082 100644 --- a/src/meshapi/tests/test_join_form.py +++ b/src/meshapi/tests/test_join_form.py @@ -186,7 +186,7 @@ def test_bad_phone_join_form(self): con = json.loads(response.content.decode("utf-8")) - self.assertEqual("555-555-5555 is not a valid phone number", con["detail"], f"Content is wrong") + self.assertEqual("555-555-5555 is not a valid phone number", con["detail"], "Content is wrong") def test_bad_email_join_form(self): # Name, email, phone, location, apt, rooftop, referral diff --git a/src/meshapi/tests/test_kml_endpoint.py b/src/meshapi/tests/test_kml_endpoint.py index 1b7345e6..022dac18 100644 --- a/src/meshapi/tests/test_kml_endpoint.py +++ b/src/meshapi/tests/test_kml_endpoint.py @@ -1,13 +1,9 @@ import datetime -import json -from django.contrib.auth.models import Group, User from django.test import Client, TestCase -from fastkml import kml from lxml import etree -from rest_framework.authtoken.models import Token -from meshapi.models import Building, Device, Install, Link, Member, Node, Sector +from meshapi.models import Building, Device, Install, Link, Member, Node def create_building_install_node_and_device(member_ref, nn): diff --git a/src/meshapi/tests/test_link.py b/src/meshapi/tests/test_link.py index 63b252a5..1e484ef4 100644 --- a/src/meshapi/tests/test_link.py +++ b/src/meshapi/tests/test_link.py @@ -3,7 +3,7 @@ from django.contrib.auth.models import User from django.test import Client, TestCase -from ..models import Building, Device, Link, Node +from ..models import Device, Link, Node from .sample_data import sample_device, sample_node diff --git a/src/meshapi/tests/test_lookups.py b/src/meshapi/tests/test_lookups.py index e49a95e1..58d4dac7 100644 --- a/src/meshapi/tests/test_lookups.py +++ b/src/meshapi/tests/test_lookups.py @@ -5,7 +5,7 @@ from django.test import Client, TestCase from ..models import Building, Device, Install, Link, Member, Node, Sector -from .sample_data import sample_building, sample_device, sample_member +from .sample_data import sample_building, sample_member class TestMemberLookups(TestCase): diff --git a/src/meshapi/tests/test_map_endpoints.py b/src/meshapi/tests/test_map_endpoints.py index b9b5fcfc..ba52e5d4 100644 --- a/src/meshapi/tests/test_map_endpoints.py +++ b/src/meshapi/tests/test_map_endpoints.py @@ -1,9 +1,7 @@ import datetime import json -from django.contrib.auth.models import Group, User from django.test import Client, TestCase -from rest_framework.authtoken.models import Token from meshapi.models import Building, Device, Install, Link, Member, Node, Sector @@ -107,7 +105,7 @@ def test_install_data(self): ap_device = Device( id=123456, node=nodes[-1], - name=f"Northwest AP", + name="Northwest AP", model="Unknown", type=Device.DeviceType.AP, install_date=datetime.date(2024, 1, 27), diff --git a/src/meshapi/tests/test_node.py b/src/meshapi/tests/test_node.py index d480cda4..9990cc5e 100644 --- a/src/meshapi/tests/test_node.py +++ b/src/meshapi/tests/test_node.py @@ -3,8 +3,8 @@ from django.contrib.auth.models import User from django.test import Client, TestCase -from ..models import Building, Device, Link, Node -from .sample_data import sample_device, sample_node +from ..models import Node +from .sample_data import sample_node class TestNode(TestCase): diff --git a/src/meshapi/tests/test_sector.py b/src/meshapi/tests/test_sector.py index 3e3258ea..1b25a74b 100644 --- a/src/meshapi/tests/test_sector.py +++ b/src/meshapi/tests/test_sector.py @@ -3,7 +3,7 @@ from django.contrib.auth.models import User from django.test import Client, TestCase -from ..models import Building, Device, Node, Sector +from ..models import Device, Node, Sector from .sample_data import sample_node diff --git a/src/meshapi/tests/test_update_panos_github.py b/src/meshapi/tests/test_update_panos_github.py index c9ef7018..0f7f3f56 100644 --- a/src/meshapi/tests/test_update_panos_github.py +++ b/src/meshapi/tests/test_update_panos_github.py @@ -4,7 +4,7 @@ from django.contrib.auth.models import User from django.test import Client, TestCase -from meshapi.models import Building, Install, Link, Member, Sector +from meshapi.models import Building, Install, Member from meshapi.views import panoramas from .sample_data import sample_building, sample_install, sample_member diff --git a/src/meshapi/tests/test_views_post_delete.py b/src/meshapi/tests/test_views_post_delete.py index 4866f570..50a68985 100644 --- a/src/meshapi/tests/test_views_post_delete.py +++ b/src/meshapi/tests/test_views_post_delete.py @@ -37,13 +37,13 @@ def test_views_post_unauthenticated(self): assert_correct_response(self, response, 403) def test_views_delete_unauthenticated(self): - response = self.c.delete(f"/api/v1/installs/1/") + response = self.c.delete("/api/v1/installs/1/") assert_correct_response(self, response, 403) - response = self.c.delete(f"/api/v1/members/1/") + response = self.c.delete("/api/v1/members/1/") assert_correct_response(self, response, 403) - response = self.c.delete(f"/api/v1/buildings/1/") + response = self.c.delete("/api/v1/buildings/1/") assert_correct_response(self, response, 403) @@ -122,13 +122,13 @@ def test_views_post_put_installer(self): assert install.notes.endswith("\n abcdef") def test_views_delete_installer(self): - response = self.c.delete(f"/api/v1/installs/1/") + response = self.c.delete("/api/v1/installs/1/") assert_correct_response(self, response, 403) - response = self.c.delete(f"/api/v1/members/1/") + response = self.c.delete("/api/v1/members/1/") assert_correct_response(self, response, 403) - response = self.c.delete(f"/api/v1/buildings/1/") + response = self.c.delete("/api/v1/buildings/1/") assert_correct_response(self, response, 403) diff --git a/src/meshapi/urls.py b/src/meshapi/urls.py index 94939cfb..e3819fee 100644 --- a/src/meshapi/urls.py +++ b/src/meshapi/urls.py @@ -1,5 +1,4 @@ -from django.urls import include, path -from rest_framework.urlpatterns import format_suffix_patterns +from django.urls import path from meshapi import views diff --git a/src/meshapi/util/django_pglocks.py b/src/meshapi/util/django_pglocks.py index 3d425129..4fbcbf76 100644 --- a/src/meshapi/util/django_pglocks.py +++ b/src/meshapi/util/django_pglocks.py @@ -12,7 +12,7 @@ @contextmanager def advisory_lock(lock_id, shared=False, wait=True, using=None): import six - from django.db import DEFAULT_DB_ALIAS, connections, transaction + from django.db import DEFAULT_DB_ALIAS, connections if using is None: using = DEFAULT_DB_ALIAS diff --git a/src/meshapi/validation.py b/src/meshapi/validation.py index 67293eda..2a397fb2 100644 --- a/src/meshapi/validation.py +++ b/src/meshapi/validation.py @@ -1,12 +1,11 @@ import json -import time -import requests from dataclasses import dataclass -from validate_email import validate_email + import phonenumbers -from geopy.geocoders import Nominatim -from meshapi.exceptions import AddressError, AddressAPIError -from meshapi.zips import NYCZipCodes +import requests +from validate_email import validate_email + +from meshapi.exceptions import AddressAPIError, AddressError from meshdb.utils.spreadsheet_import.building.constants import INVALID_BIN_NUMBERS from meshdb.utils.spreadsheet_import.building.pelias import humanify_street_address @@ -61,7 +60,7 @@ def __init__(self, street_address: str, city: str, state: str, zip: int): "text": self.address, "size": 1, } - nyc_planning_req = requests.get(f"https://geosearch.planninglabs.nyc/v2/search", params=query_params) + nyc_planning_req = requests.get("https://geosearch.planninglabs.nyc/v2/search", params=query_params) nyc_planning_resp = json.loads(nyc_planning_req.content.decode("utf-8")) except Exception as e: print(f"Got exception querying geosearch.planninglabs.nyc: {e}") @@ -107,7 +106,7 @@ def __init__(self, street_address: str, city: str, state: str, zip: int): "$select": "heightroof,groundelev", "$limit": 1, } - nyc_dataset_req = requests.get(f"https://data.cityofnewyork.us/resource/qb5r-6dgf.json", params=query_params) + nyc_dataset_req = requests.get("https://data.cityofnewyork.us/resource/qb5r-6dgf.json", params=query_params) nyc_dataset_resp = json.loads(nyc_dataset_req.content.decode("utf-8")) if len(nyc_dataset_resp) == 0: diff --git a/src/meshapi/views/__init__.py b/src/meshapi/views/__init__.py index 8e14b870..9768ec0c 100644 --- a/src/meshapi/views/__init__.py +++ b/src/meshapi/views/__init__.py @@ -3,5 +3,5 @@ from .lookups import * from .map import * from .model_api import * -from .query_api import * from .panoramas import * +from .query_api import * diff --git a/src/meshapi/views/forms.py b/src/meshapi/views/forms.py index 0b8a17d6..0b939b24 100644 --- a/src/meshapi/views/forms.py +++ b/src/meshapi/views/forms.py @@ -2,7 +2,7 @@ import logging import time from dataclasses import dataclass -from datetime import date, datetime +from datetime import date from json.decoder import JSONDecodeError from typing import Optional @@ -127,7 +127,7 @@ def join_form(request): print("(NYC) Something went wrong validating the address. Re-trying...") time.sleep(3) # If we run out of tries, bail. - if nyc_addr_info == None: + if nyc_addr_info is None: print(f"Could not parse address: {r.street_address}, {r.city}, {r.state}, {r.zip}") return Response( {"detail": "Your address could not be validated."}, status=status.HTTP_500_INTERNAL_SERVER_ERROR @@ -401,7 +401,7 @@ def network_number_assignment(request): return Response({"detail": "Install Number not found"}, status=status.HTTP_404_NOT_FOUND) # Check if the install already has a network number - if nn_install.node != None: + if nn_install.node is not None: message = f"This Install Number ({r.install_number}) already has a Network Number ({nn_install.node.network_number}) associated with it!" print(message) return Response( diff --git a/src/meshapi/views/geography.py b/src/meshapi/views/geography.py index 82ab1619..efcf243c 100644 --- a/src/meshapi/views/geography.py +++ b/src/meshapi/views/geography.py @@ -12,7 +12,7 @@ from rest_framework.negotiation import BaseContentNegotiation from rest_framework.views import APIView -from meshapi.models import Building, Install, Link, Node +from meshapi.models import Install, Link KML_CONTENT_TYPE = "application/vnd.google-earth.kml+xml" KML_CONTENT_TYPE_WITH_CHARSET = f"{KML_CONTENT_TYPE}; charset=utf-8" diff --git a/src/meshapi/views/map.py b/src/meshapi/views/map.py index 065d0dc7..7249f9c2 100644 --- a/src/meshapi/views/map.py +++ b/src/meshapi/views/map.py @@ -4,7 +4,7 @@ from drf_spectacular.utils import extend_schema, extend_schema_view from rest_framework import generics, permissions -from meshapi.models import Building, Device, Install, Link, Node, Sector +from meshapi.models import Device, Install, Link, Node, Sector from meshapi.serializers import ( ALLOWED_INSTALL_STATUSES, EXCLUDED_INSTALL_STATUSES, diff --git a/src/meshapi/views/model_api.py b/src/meshapi/views/model_api.py index 0ae328fe..ae95e4ae 100644 --- a/src/meshapi/views/model_api.py +++ b/src/meshapi/views/model_api.py @@ -1,4 +1,3 @@ -from django.contrib.auth.models import User from drf_spectacular.types import OpenApiTypes from drf_spectacular.utils import OpenApiExample, OpenApiResponse, extend_schema, extend_schema_view from rest_framework import generics, permissions @@ -6,7 +5,6 @@ from rest_framework.response import Response from meshapi.models import Building, Device, Install, Link, Member, Node, Sector -from meshapi.permissions import IsReadOnly from meshapi.serializers import ( BuildingSerializer, DeviceSerializer, diff --git a/src/meshapi/views/panoramas.py b/src/meshapi/views/panoramas.py index 2aac308f..b533294a 100644 --- a/src/meshapi/views/panoramas.py +++ b/src/meshapi/views/panoramas.py @@ -1,17 +1,16 @@ import os from pathlib import Path + import requests -from rest_framework import permissions +from celery.schedules import crontab from rest_framework.decorators import api_view, permission_classes from rest_framework.response import Response from rest_framework.views import status -from meshapi.models import Building, Install -from meshapi.permissions import HasPanoramaUpdatePermission +from meshapi.models import Install +from meshapi.permissions import HasPanoramaUpdatePermission from meshapi.util.django_pglocks import advisory_lock - from meshdb.celery import app as celery_app -from celery.schedules import crontab # Config for gathering/generating panorama links PANO_REPO_OWNER = "nycmeshnet" diff --git a/src/meshapi/views/query_api.py b/src/meshapi/views/query_api.py index 19ae2edf..5e1daf32 100644 --- a/src/meshapi/views/query_api.py +++ b/src/meshapi/views/query_api.py @@ -1,26 +1,12 @@ -import os from typing import Any, Dict, Optional -from drf_spectacular.utils import extend_schema, extend_schema_view -from meshapi.views.lookups import FilterRequiredListAPIView, InstallFilter -from rest_framework import generics -from rest_framework.views import models - -from meshapi import permissions -from meshapi.docs import map_query_filters_to_param_annotations, query_form_password_param -from meshapi.models import Building, Install, Member -from meshapi.serializers.query_api import QueryFormSerializer from django.db.models import Q from django_filters import rest_framework as filters -from drf_spectacular.types import OpenApiTypes -from drf_spectacular.utils import OpenApiParameter, extend_schema, extend_schema_view -from rest_framework import generics -from rest_framework.response import Response -from rest_framework.decorators import api_view, permission_classes - -from rest_framework import permissions +from meshapi.models import Install from meshapi.permissions import LegacyMeshQueryPassword +from meshapi.serializers.query_api import QueryFormSerializer +from meshapi.views.lookups import FilterRequiredListAPIView """ Re-implements https://docs.nycmesh.net/installs/query/ diff --git a/src/meshapi_hooks/migrations/0001_initial.py b/src/meshapi_hooks/migrations/0001_initial.py index d7c30661..d856accf 100644 --- a/src/meshapi_hooks/migrations/0001_initial.py +++ b/src/meshapi_hooks/migrations/0001_initial.py @@ -1,9 +1,9 @@ # Generated by Django 4.2.9 on 2024-02-06 02:26 -from django.conf import settings -from django.db import migrations, models import django.db.models.deletion import drf_hooks.models +from django.conf import settings +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/src/meshapi_hooks/migrations/0002_celeryserializerhook_and_more.py b/src/meshapi_hooks/migrations/0002_celeryserializerhook_and_more.py index c3ed1afc..c2994f44 100644 --- a/src/meshapi_hooks/migrations/0002_celeryserializerhook_and_more.py +++ b/src/meshapi_hooks/migrations/0002_celeryserializerhook_and_more.py @@ -1,9 +1,9 @@ # Generated by Django 4.2.9 on 2024-02-19 00:54 -from django.conf import settings -from django.db import migrations, models import django.db.models.deletion import drf_hooks.models +from django.conf import settings +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/src/meshapi_hooks/tests/test_webhooks.py b/src/meshapi_hooks/tests/test_webhooks.py index c676cf26..ea98f82c 100644 --- a/src/meshapi_hooks/tests/test_webhooks.py +++ b/src/meshapi_hooks/tests/test_webhooks.py @@ -175,7 +175,7 @@ def test_webhook_gets_disabled_after_many_retries(self): timeout=HTTP_CALL_WAITING_TIME * 5, # Wait extra time before giving up, for the retries to happen ) assert False, "/webhook HTTP endpoint shouldn't have been called (only /bad-webhook)" - except queue.Empty as e: + except queue.Empty: pass # We have failed once, check that we have recorded this failure in the hook object @@ -196,7 +196,7 @@ def test_webhook_gets_disabled_after_many_retries(self): timeout=HTTP_CALL_WAITING_TIME * 5, # Wait extra time before giving up, for the retries to happen ) assert False, "/webhook HTTP endpoint shouldn't have been called (only /bad-webhook)" - except queue.Empty as e: + except queue.Empty: pass # We have failed once, check that we have recorded this failure in the hook object @@ -234,7 +234,7 @@ def test_install(self): # so they should be able to see the email, etc assert flask_request["data"]["member"] == self.member_obj.id assert flask_request["data"]["install_number"] == install_obj.install_number - assert flask_request["data"]["network_number"] == None + assert flask_request["data"]["network_number"] is None assert flask_request["hook"]["event"] == "install.created" assert flask_request["hook"]["target"] == "http://localhost:8091/webhook" diff --git a/src/meshdb/urls.py b/src/meshdb/urls.py index 1e2add0c..bcd6f750 100644 --- a/src/meshdb/urls.py +++ b/src/meshdb/urls.py @@ -14,9 +14,10 @@ 1. Import the include() function: from django.urls import include, path 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ + from django.contrib import admin from django.urls import include, path -from drf_spectacular.views import SpectacularAPIView, SpectacularRedocView, SpectacularSwaggerView +from drf_spectacular.views import SpectacularAPIView, SpectacularRedocView from meshapi.docs import SpectacularSwaggerInjectVarsView from meshdb.settings import PROFILING_ENABLED diff --git a/src/meshdb/utils/spreadsheet_import/building/resolve_address.py b/src/meshdb/utils/spreadsheet_import/building/resolve_address.py index 47faf295..d50c6efe 100644 --- a/src/meshdb/utils/spreadsheet_import/building/resolve_address.py +++ b/src/meshdb/utils/spreadsheet_import/building/resolve_address.py @@ -11,8 +11,6 @@ from meshdb.utils.spreadsheet_import.building.constants import ( INVALID_BIN_NUMBERS, LOCAL_MESH_NOMINATIM_ADDR, - NYC_BIN_LOOKUP_PREFIX, - NYC_BIN_LOOKUP_UA, NYC_COUNTIES, OSM_CITY_SUBSTITUTIONS, AddressParsingResult, @@ -67,13 +65,17 @@ def convert_osm_city_village_suburb_nonsense(osm_raw_addr: dict) -> Tuple[str, s ) return ( - osm_raw_addr["city"] - if "city" in osm_raw_addr - else osm_raw_addr["town"] - if "town" in osm_raw_addr - else osm_raw_addr["village"] - if "village" in osm_raw_addr - else None, + ( + osm_raw_addr["city"] + if "city" in osm_raw_addr + else ( + osm_raw_addr["town"] + if "town" in osm_raw_addr + else osm_raw_addr["village"] + if "village" in osm_raw_addr + else None + ) + ), osm_raw_addr["ISO3166-2-lvl4"].split("-")[1] if "ISO3166-2-lvl4" in osm_raw_addr else None, ) @@ -227,7 +229,7 @@ def _find_nyc_building( # Empirically, the /autocomplete endpoint performs better than the /search endpoint # (don't ask me why) - nyc_planning_req = requests.get(f"https://geosearch.planninglabs.nyc/v2/autocomplete", params=query_params) + nyc_planning_req = requests.get("https://geosearch.planninglabs.nyc/v2/autocomplete", params=query_params) nyc_planning_resp = nyc_planning_req.json() closest_nyc_location = None @@ -293,75 +295,6 @@ def _find_nyc_building( ) output_bin = new_bin - # if new_bin != spreadsheet_bin and spreadsheet_bin not in INVALID_BIN_NUMBERS: - # new_bin_lookup_response = requests.get( - # NYC_BIN_LOOKUP_PREFIX + str(new_bin), - # headers={"User-Agent": NYC_BIN_LOOKUP_UA}, - # ).json() - # new_alternate_bins = ( - # (new_bin_lookup_response["PropertyDetails"]["AdditionalBINsforBuilding"] or "") - # if new_bin_lookup_response["PropertyDetails"] - # else "" - # ).split(", ") - # new_bin_housenum = normalize_whitespace_and_case( - # new_bin_lookup_response["PropertyDetails"]["HouseNo"] - # if new_bin_lookup_response["PropertyDetails"] - # else "" - # ) - # new_bin_street = normalize_whitespace_and_case( - # new_bin_lookup_response["PropertyDetails"]["StreetName"] - # if new_bin_lookup_response["PropertyDetails"] - # else "" - # ) - # - # old_bin_lookup_response = requests.get( - # NYC_BIN_LOOKUP_PREFIX + str(spreadsheet_bin), - # headers={"User-Agent": NYC_BIN_LOOKUP_UA}, - # ).json() - # old_alternate_bins = ( - # (old_bin_lookup_response["PropertyDetails"]["AdditionalBINsforBuilding"] or "") - # if old_bin_lookup_response["PropertyDetails"] - # else "" - # ).split(", ") - # old_bin_housenum = normalize_whitespace_and_case( - # old_bin_lookup_response["PropertyDetails"]["HouseNo"] - # if old_bin_lookup_response["PropertyDetails"] - # else "" - # ) - # old_bin_street = normalize_whitespace_and_case( - # old_bin_lookup_response["PropertyDetails"]["StreetName"] - # if old_bin_lookup_response["PropertyDetails"] - # else "" - # ) - # old_bin_zip = normalize_whitespace_and_case( - # old_bin_lookup_response["PropertyDetails"]["Zip"] if old_bin_lookup_response["PropertyDetails"] else "" - # ) - # - # if str(new_bin) in old_alternate_bins or str(spreadsheet_bin) in new_alternate_bins: - # # Some buildings have more than one BIN, if this explains the mismatch, - # # it's more or less impossible to tell which one we should be using, We will - # # arbitrarily pick the one we just got back since it's "more recent" and quietly - # # use that one - # output_bin = new_bin - # elif ( - # old_bin_housenum != normalize_whitespace_and_case(closest_nyc_location["properties"]["housenumber"]) - # or old_bin_street != normalize_whitespace_and_case(closest_nyc_location["properties"]["street"]) - # or old_bin_zip != zip_code - # ): - # # The old BIN doesn't look right, maybe the spreadsheet is incorrect or out of date? - # # Let's quietly use the new BIN - # output_bin = new_bin - # else: - # # Otherwise, this is not easily explicable, warn the user about it - # # TODO: Should we flag as invalid in this case? - # logging.warning( - # f"BIN mismatch for '{normalized_nyc_addr}'. " - # f"Spreadsheet uses {spreadsheet_bin}, but we found {output_bin} " - # f"for this address. We checked to see if this building has more than one BIN, " - # f"or if the old BIN was incorrect, but couldn't explain the discrepancy " - # f"using these methods. We're going to assume that our search was correct and " - # f"use {output_bin} in the database." - # ) return AddressParsingResult( address=DatabaseAddress(street_address, city, state, zip_code), diff --git a/src/meshdb/utils/spreadsheet_import/csv_load.py b/src/meshdb/utils/spreadsheet_import/csv_load.py index 1d976d3c..800d5983 100644 --- a/src/meshdb/utils/spreadsheet_import/csv_load.py +++ b/src/meshdb/utils/spreadsheet_import/csv_load.py @@ -238,7 +238,7 @@ def print_dropped_edit_report( "ModifiedProperty", "DatabaseValue", "DroppedValue", - ] + ], # + list(csv_reader.fieldnames), ) csv_writer.writeheader() diff --git a/src/meshdb/utils/spreadsheet_import/main.py b/src/meshdb/utils/spreadsheet_import/main.py index fbe38956..6bc736fd 100644 --- a/src/meshdb/utils/spreadsheet_import/main.py +++ b/src/meshdb/utils/spreadsheet_import/main.py @@ -19,8 +19,6 @@ from meshdb.utils.spreadsheet_import.building.resolve_address import AddressParser from meshdb.utils.spreadsheet_import.csv_load import ( DroppedModification, - SpreadsheetLinkStatus, - SpreadsheetSectorStatus, get_spreadsheet_links, get_spreadsheet_rows, get_spreadsheet_sectors, @@ -29,7 +27,7 @@ ) from meshdb.utils.spreadsheet_import.parse_building import get_or_create_building from meshdb.utils.spreadsheet_import.parse_install import create_install, normalize_install_to_primary_building_node -from meshdb.utils.spreadsheet_import.parse_link import create_link, load_links_supplement_with_uisp +from meshdb.utils.spreadsheet_import.parse_link import load_links_supplement_with_uisp from meshdb.utils.spreadsheet_import.parse_member import get_or_create_member from meshdb.utils.spreadsheet_import.parse_node import get_or_create_node, normalize_building_node_links diff --git a/src/meshdb/utils/spreadsheet_import/parse_building.py b/src/meshdb/utils/spreadsheet_import/parse_building.py index c1c589d0..3951304a 100644 --- a/src/meshdb/utils/spreadsheet_import/parse_building.py +++ b/src/meshdb/utils/spreadsheet_import/parse_building.py @@ -6,7 +6,7 @@ from meshapi import models from meshapi.exceptions import AddressError from meshapi.models import Building -from meshdb.utils.spreadsheet_import.building.constants import INVALID_BIN_NUMBERS, AddressTruthSource, DatabaseAddress +from meshdb.utils.spreadsheet_import.building.constants import INVALID_BIN_NUMBERS, DatabaseAddress from meshdb.utils.spreadsheet_import.building.resolve_address import AddressParser from meshdb.utils.spreadsheet_import.csv_load import DroppedModification, SpreadsheetRow @@ -174,9 +174,11 @@ def get_or_create_building( DroppedModification( [row.id], row.id, - address_result.discovered_bin - if address_result.discovered_bin - else address_result.address.street_address, + ( + address_result.discovered_bin + if address_result.discovered_bin + else address_result.address.street_address + ), "lat_long_discrepancy_vs_spreadsheet", str(address_result.discovered_lat_lon), str((row.latitude, row.longitude)), diff --git a/src/meshdb/utils/spreadsheet_import/parse_devices.py b/src/meshdb/utils/spreadsheet_import/parse_devices.py index da0bf8fd..ace7aea1 100644 --- a/src/meshdb/utils/spreadsheet_import/parse_devices.py +++ b/src/meshdb/utils/spreadsheet_import/parse_devices.py @@ -19,7 +19,6 @@ SpreadsheetSector, SpreadsheetSectorStatus, SpreadsheetStatus, - get_spreadsheet_rows, get_spreadsheet_sectors, ) from meshdb.utils.spreadsheet_import.fetch_uisp import download_uisp_devices @@ -242,9 +241,11 @@ def load_access_points(spreadsheet_installs: List[SpreadsheetRow]): name=f"{row.nodeName} AP" if row.nodeName else "AP", model="Unknown", type=Device.DeviceType.AP, - status=Device.DeviceStatus.ACTIVE - if row.status == SpreadsheetStatus.installed - else Device.DeviceStatus.INACTIVE, + status=( + Device.DeviceStatus.ACTIVE + if row.status == SpreadsheetStatus.installed + else Device.DeviceStatus.INACTIVE + ), latitude=row.latitude, longitude=row.longitude, install_date=row.installDate, diff --git a/src/meshdb/utils/spreadsheet_import/parse_link.py b/src/meshdb/utils/spreadsheet_import/parse_link.py index dbc08a60..d6e6c097 100644 --- a/src/meshdb/utils/spreadsheet_import/parse_link.py +++ b/src/meshdb/utils/spreadsheet_import/parse_link.py @@ -12,9 +12,8 @@ from meshapi import models from meshapi.models import Device, Install, Link, Node -from meshdb.utils.spreadsheet_import.building.lookup import get_building_from_node_id from meshdb.utils.spreadsheet_import.csv_load import SpreadsheetLink, SpreadsheetLinkStatus, get_spreadsheet_links -from meshdb.utils.spreadsheet_import.fetch_uisp import download_uisp_devices, download_uisp_links +from meshdb.utils.spreadsheet_import.fetch_uisp import download_uisp_links def convert_spreadsheet_link_type(status: SpreadsheetLinkStatus) -> Link.LinkType: diff --git a/src/meshdb/utils/spreadsheet_import/tests/test_humanify_addr.py b/src/meshdb/utils/spreadsheet_import/tests/test_humanify_addr.py index 65cfb698..80dca4d5 100644 --- a/src/meshdb/utils/spreadsheet_import/tests/test_humanify_addr.py +++ b/src/meshdb/utils/spreadsheet_import/tests/test_humanify_addr.py @@ -1,5 +1,3 @@ -import pytest - from meshdb.utils.spreadsheet_import.building.pelias import humanify_street_address diff --git a/src/meshweb/tests.py b/src/meshweb/tests.py index 7ce503c2..a79ca8be 100644 --- a/src/meshweb/tests.py +++ b/src/meshweb/tests.py @@ -1,3 +1,3 @@ -from django.test import TestCase +# from django.test import TestCase # Create your tests here. diff --git a/src/meshweb/urls.py b/src/meshweb/urls.py index 20622e84..609af97e 100644 --- a/src/meshweb/urls.py +++ b/src/meshweb/urls.py @@ -1,4 +1,5 @@ -from django.urls import path, include +from django.urls import path + from meshweb import views urlpatterns = [ diff --git a/tasks.py b/tasks.py index e5f954da..6b49ad08 100644 --- a/tasks.py +++ b/tasks.py @@ -4,14 +4,14 @@ @task def format(context): context.run("black .") - # context.run("isort .") + context.run("isort .") @task def lint(context): context.run("black . --check") - # context.run("isort . --check") - # context.run("flake8 meshdb tests unit_tests") + context.run("isort . --check") + # context.run("flake8 src") # context.run("mypy meshdb")