From 5783c0010774b78ce75a179ddb14fcd3cf3303ac Mon Sep 17 00:00:00 2001 From: Andrew Dickinson Date: Thu, 15 Feb 2024 20:53:01 -0500 Subject: [PATCH] Fix: wrong building is attached to imported links in some cases (#191) * Refactor Link and Sector import into separate files like the other models * Modify & refactor logic to get building from node ID --- .../spreadsheet_import/building/lookup.py | 17 +++ .../utils/spreadsheet_import/csv_load.py | 8 +- src/meshdb/utils/spreadsheet_import/main.py | 106 ++---------------- .../utils/spreadsheet_import/parse_link.py | 57 ++++++++++ .../utils/spreadsheet_import/parse_sector.py | 46 ++++++++ 5 files changed, 132 insertions(+), 102 deletions(-) create mode 100644 src/meshdb/utils/spreadsheet_import/building/lookup.py create mode 100644 src/meshdb/utils/spreadsheet_import/parse_link.py create mode 100644 src/meshdb/utils/spreadsheet_import/parse_sector.py diff --git a/src/meshdb/utils/spreadsheet_import/building/lookup.py b/src/meshdb/utils/spreadsheet_import/building/lookup.py new file mode 100644 index 00000000..ed724222 --- /dev/null +++ b/src/meshdb/utils/spreadsheet_import/building/lookup.py @@ -0,0 +1,17 @@ +from meshapi import models + + +def get_building_from_node_id(node_id: int) -> models.Building: + install_obj = models.Install.objects.filter(install_number=node_id).first() + if install_obj and install_obj.InstallStatus != models.Install.InstallStatus.NN_REASSIGNED: + return install_obj.building + + install_obj = models.Install.objects.filter(network_number=node_id).first() + if install_obj: + return install_obj.building + + building_obj = models.Building.objects.filter(primary_nn=node_id).first() + if building_obj: + return building_obj + + raise ValueError(f"Could not find building for install #{node_id}") diff --git a/src/meshdb/utils/spreadsheet_import/csv_load.py b/src/meshdb/utils/spreadsheet_import/csv_load.py index 4a4cb0c7..1abb5f23 100644 --- a/src/meshdb/utils/spreadsheet_import/csv_load.py +++ b/src/meshdb/utils/spreadsheet_import/csv_load.py @@ -71,8 +71,8 @@ class SpreadsheetRow: @dataclasses.dataclass class SpreadsheetLink: - from_install_num: int - to_install_num: int + from_node_id: int + to_node_id: int status: SpreadsheetLinkStatus install_date: Optional[datetime.date] abandon_date: Optional[datetime.date] @@ -280,8 +280,8 @@ def get_spreadsheet_links(links_path: str) -> List[SpreadsheetLink]: links.append( SpreadsheetLink( - from_install_num=int(row["from"]), - to_install_num=int(row["to"]), + from_node_id=int(row["from"]), + to_node_id=int(row["to"]), status=SpreadsheetLinkStatus(row["status"]), install_date=install_date, abandon_date=abandon_date, diff --git a/src/meshdb/utils/spreadsheet_import/main.py b/src/meshdb/utils/spreadsheet_import/main.py index 5012ee1d..54f1cecf 100644 --- a/src/meshdb/utils/spreadsheet_import/main.py +++ b/src/meshdb/utils/spreadsheet_import/main.py @@ -6,7 +6,6 @@ from typing import List import django -from django.db.models import Q from meshdb.utils.spreadsheet_import import logger @@ -29,7 +28,9 @@ ) from meshdb.utils.spreadsheet_import.parse_building import get_or_create_building from meshdb.utils.spreadsheet_import.parse_install import get_or_create_install +from meshdb.utils.spreadsheet_import.parse_link import create_link from meshdb.utils.spreadsheet_import.parse_member import get_or_create_member +from meshdb.utils.spreadsheet_import.parse_sector import create_sector def main(): @@ -119,107 +120,16 @@ def main(): logging.info(f"Loading links from '{links_path}'...") links = get_spreadsheet_links(links_path) for spreadsheet_link in links: - try: - from_building = models.Building.objects.filter( - Q(installs__install_number=spreadsheet_link.from_install_num) - | Q(installs__network_number=spreadsheet_link.from_install_num) - | Q(primary_nn=spreadsheet_link.from_install_num), - )[0] - to_building = models.Building.objects.filter( - Q(installs__install_number=spreadsheet_link.to_install_num) - | Q(installs__network_number=spreadsheet_link.to_install_num) - | Q(primary_nn=spreadsheet_link.to_install_num), - )[0] - except IndexError: - message = ( - f"Could not find building for install {spreadsheet_link.from_install_num} or " - f"{spreadsheet_link.to_install_num}" - ) - if spreadsheet_link.status != SpreadsheetLinkStatus.dead: - raise ValueError(message) - else: - logging.warning(message + ". But this link is dead, skipping this spreadsheet row") - continue - - if spreadsheet_link.status in [ - SpreadsheetLinkStatus.vpn, - SpreadsheetLinkStatus.active, - SpreadsheetLinkStatus.sixty_ghz, - SpreadsheetLinkStatus.fiber, - ]: - status = models.Link.LinkStatus.ACTIVE - elif spreadsheet_link.status == SpreadsheetLinkStatus.dead: - status = models.Link.LinkStatus.DEAD - elif spreadsheet_link.status == SpreadsheetLinkStatus.planned: - status = models.Link.LinkStatus.PLANNED - else: - raise ValueError(f"Invalid spreadsheet link status {spreadsheet_link.status}") - - link_type = None - if spreadsheet_link.status in [ - SpreadsheetLinkStatus.active, - ]: - link_type = models.Link.LinkType.STANDARD - elif spreadsheet_link.status == SpreadsheetLinkStatus.vpn: - link_type = models.Link.LinkType.VPN - elif spreadsheet_link.status == SpreadsheetLinkStatus.sixty_ghz: - link_type = models.Link.LinkType.MMWAVE - elif spreadsheet_link.status == SpreadsheetLinkStatus.fiber: - link_type = models.Link.LinkType.FIBER - - link_notes = "\n".join([spreadsheet_link.notes, spreadsheet_link.comments]).strip() - link = models.Link( - from_building=from_building, - to_building=to_building, - status=status, - type=link_type, - install_date=spreadsheet_link.install_date, - abandon_date=spreadsheet_link.abandon_date, - description=spreadsheet_link.where_to_where if spreadsheet_link.where_to_where else None, - notes=link_notes if link_notes else None, - ) - link.save() + link = create_link(spreadsheet_link) + if link: + link.save() logging.info(f"Loading sectors from '{sectors_path}'...") sectors = get_spreadsheet_sectors(sectors_path) for spreadsheet_sector in sectors: - try: - building = models.Building.objects.filter( - Q(installs__install_number=spreadsheet_sector.node_id) - | Q(installs__network_number=spreadsheet_sector.node_id) - | Q(primary_nn=spreadsheet_sector.node_id), - )[0] - except IndexError: - message = f"Could not find building for install {spreadsheet_sector.node_id}" - if spreadsheet_sector.status != SpreadsheetSectorStatus.abandoned: - raise ValueError(message) - else: - logging.warning(message + ". But this sector is abandoned, skipping this spreadsheet row") - - if spreadsheet_sector.status == SpreadsheetSectorStatus.active: - status = models.Sector.SectorStatus.ACTIVE - elif spreadsheet_sector.status == SpreadsheetSectorStatus.abandoned: - status = models.Sector.SectorStatus.ABANDONED - elif spreadsheet_sector.status == SpreadsheetSectorStatus.potential: - status = models.Sector.SectorStatus.POTENTIAL - else: - raise ValueError(f"Invalid spreadsheet sector status {spreadsheet_sector.status}") - - sector_notes = "\n".join([spreadsheet_sector.notes, spreadsheet_sector.comments]).strip() - sector = models.Sector( - building=building, - radius=spreadsheet_sector.radius, - azimuth=spreadsheet_sector.azimuth, - width=spreadsheet_sector.width, - status=status, - install_date=spreadsheet_sector.install_date, - abandon_date=spreadsheet_sector.abandon_date, - device_name=spreadsheet_sector.device, - name=spreadsheet_sector.names, - ssid=spreadsheet_sector.ssid if spreadsheet_sector.ssid else None, - notes=sector_notes if sector_notes else None, - ) - sector.save() + sector = create_sector(spreadsheet_sector) + if sector: + sector.save() if __name__ == "__main__": diff --git a/src/meshdb/utils/spreadsheet_import/parse_link.py b/src/meshdb/utils/spreadsheet_import/parse_link.py new file mode 100644 index 00000000..35bef486 --- /dev/null +++ b/src/meshdb/utils/spreadsheet_import/parse_link.py @@ -0,0 +1,57 @@ +import logging +from typing import Optional + +from meshapi import models +from meshdb.utils.spreadsheet_import.building.lookup import get_building_from_node_id +from meshdb.utils.spreadsheet_import.csv_load import SpreadsheetLink, SpreadsheetLinkStatus + + +def create_link(spreadsheet_link: SpreadsheetLink) -> Optional[models.Link]: + try: + from_building = get_building_from_node_id(spreadsheet_link.from_node_id) + to_building = get_building_from_node_id(spreadsheet_link.to_node_id) + except ValueError as e: + if spreadsheet_link.status != SpreadsheetLinkStatus.dead: + raise e + else: + logging.warning(f"Error while parsing links: { e }. But this link is dead, skipping this spreadsheet row") + return None + + if spreadsheet_link.status in [ + SpreadsheetLinkStatus.vpn, + SpreadsheetLinkStatus.active, + SpreadsheetLinkStatus.sixty_ghz, + SpreadsheetLinkStatus.fiber, + ]: + status = models.Link.LinkStatus.ACTIVE + elif spreadsheet_link.status == SpreadsheetLinkStatus.dead: + status = models.Link.LinkStatus.DEAD + elif spreadsheet_link.status == SpreadsheetLinkStatus.planned: + status = models.Link.LinkStatus.PLANNED + else: + raise ValueError(f"Invalid spreadsheet link status {spreadsheet_link.status}") + + link_type = None + if spreadsheet_link.status in [ + SpreadsheetLinkStatus.active, + ]: + link_type = models.Link.LinkType.STANDARD + elif spreadsheet_link.status == SpreadsheetLinkStatus.vpn: + link_type = models.Link.LinkType.VPN + elif spreadsheet_link.status == SpreadsheetLinkStatus.sixty_ghz: + link_type = models.Link.LinkType.MMWAVE + elif spreadsheet_link.status == SpreadsheetLinkStatus.fiber: + link_type = models.Link.LinkType.FIBER + + link_notes = "\n".join([spreadsheet_link.notes, spreadsheet_link.comments]).strip() + link = models.Link( + from_building=from_building, + to_building=to_building, + status=status, + type=link_type, + install_date=spreadsheet_link.install_date, + abandon_date=spreadsheet_link.abandon_date, + description=spreadsheet_link.where_to_where if spreadsheet_link.where_to_where else None, + notes=link_notes if link_notes else None, + ) + return link diff --git a/src/meshdb/utils/spreadsheet_import/parse_sector.py b/src/meshdb/utils/spreadsheet_import/parse_sector.py new file mode 100644 index 00000000..8d58c123 --- /dev/null +++ b/src/meshdb/utils/spreadsheet_import/parse_sector.py @@ -0,0 +1,46 @@ +import logging +from typing import Optional + +from django.db.models import Q + +from meshapi import models +from meshdb.utils.spreadsheet_import.building.lookup import get_building_from_node_id +from meshdb.utils.spreadsheet_import.csv_load import SpreadsheetSector, SpreadsheetSectorStatus + + +def create_sector(spreadsheet_sector: SpreadsheetSector) -> Optional[models.Sector]: + try: + building = get_building_from_node_id(spreadsheet_sector.node_id) + except ValueError as e: + if spreadsheet_sector.status != SpreadsheetSectorStatus.abandoned: + raise e + else: + logging.warning( + f"Error while parsing sectors: {e}. But this sector is abandoned, skipping this spreadsheet row" + ) + return None + + if spreadsheet_sector.status == SpreadsheetSectorStatus.active: + status = models.Sector.SectorStatus.ACTIVE + elif spreadsheet_sector.status == SpreadsheetSectorStatus.abandoned: + status = models.Sector.SectorStatus.ABANDONED + elif spreadsheet_sector.status == SpreadsheetSectorStatus.potential: + status = models.Sector.SectorStatus.POTENTIAL + else: + raise ValueError(f"Invalid spreadsheet sector status {spreadsheet_sector.status}") + + sector_notes = "\n".join([spreadsheet_sector.notes, spreadsheet_sector.comments]).strip() + sector = models.Sector( + building=building, + radius=spreadsheet_sector.radius, + azimuth=spreadsheet_sector.azimuth, + width=spreadsheet_sector.width, + status=status, + install_date=spreadsheet_sector.install_date, + abandon_date=spreadsheet_sector.abandon_date, + device_name=spreadsheet_sector.device, + name=spreadsheet_sector.names, + ssid=spreadsheet_sector.ssid if spreadsheet_sector.ssid else None, + notes=sector_notes if sector_notes else None, + ) + return sector