diff --git a/src/meshdb/utils/spreadsheet_import/csv_load.py b/src/meshdb/utils/spreadsheet_import/csv_load.py index 800d5983..519aba5c 100644 --- a/src/meshdb/utils/spreadsheet_import/csv_load.py +++ b/src/meshdb/utils/spreadsheet_import/csv_load.py @@ -101,6 +101,7 @@ class SpreadsheetSector: class DroppedModification: original_row_ids: List[int] new_row_id: int + row_status: str deduplication_value: str modified_property: str database_value: str @@ -109,12 +110,13 @@ class DroppedModification: def get_spreadsheet_rows( form_responses_path: str, -) -> Tuple[List[SpreadsheetRow], Dict[int, str]]: +) -> Tuple[List[SpreadsheetRow], List[SpreadsheetRow], Dict[int, str]]: with open(form_responses_path, "r") as input_file: csv_reader = csv.DictReader(input_file) skipped_rows: Dict[int, str] = {} - nodes: List[SpreadsheetRow] = [] + installs: List[SpreadsheetRow] = [] + reassigned_nns: List[SpreadsheetRow] = [] for i, row in enumerate(csv_reader): # Last row is placeholder @@ -140,11 +142,16 @@ def get_spreadsheet_rows( except ValueError: abandon_date = None + node_status = SpreadsheetStatus(row["Status"].replace("dupe", "Dupe")) + re_assigned_as_nn = False try: nn = row["NN"].lower().strip() - re_assigned_as_nn = nn.startswith("x-") - nn = int(nn) if nn is not None and nn != "" and not re_assigned_as_nn else None + re_assigned_as_nn = nn.startswith("x-") or node_status == SpreadsheetStatus.nnAssigned + if re_assigned_as_nn: + nn = node_id + else: + nn = int(nn) if nn is not None and nn != "" else None except (KeyError, ValueError): nn = None @@ -159,7 +166,7 @@ def get_spreadsheet_rows( secondEmail=row["2nd profile email"].lower().strip(), phone=row["Phone"], roofAccess=row["Rooftop Access"] == "I have Rooftop access", - status=SpreadsheetStatus(row["Status"].replace("dupe", "Dupe")), + status=node_status, installDate=install_date, abandonDate=abandon_date, nodeName=row["nodeName"], @@ -181,12 +188,12 @@ def get_spreadsheet_rows( continue if re_assigned_as_nn: - skipped_rows[node_id] = "Reassigned as NN for another row" + reassigned_nns.append(node) continue - nodes.append(node) + installs.append(node) - return nodes, skipped_rows + return installs, reassigned_nns, skipped_rows def print_failure_report(skipped_rows: Dict[int, str], original_input_file: str, fname_overide: str = None) -> None: @@ -234,6 +241,7 @@ def print_dropped_edit_report( [ "OriginalRowID(s)", "DroppedRowID", + "DroppedRowSpreadsheetStatus", "DeduplicationValue", "ModifiedProperty", "DatabaseValue", @@ -249,6 +257,7 @@ def print_dropped_edit_report( new_fields = {} new_fields["OriginalRowID(s)"] = ", ".join(str(row_id) for row_id in edit.original_row_ids) new_fields["DroppedRowID"] = edit.new_row_id + new_fields["DroppedRowSpreadsheetStatus"] = edit.row_status new_fields["DeduplicationValue"] = edit.deduplication_value new_fields["ModifiedProperty"] = edit.modified_property new_fields["DatabaseValue"] = edit.database_value diff --git a/src/meshdb/utils/spreadsheet_import/main.py b/src/meshdb/utils/spreadsheet_import/main.py index 6bc736fd..75e0360f 100644 --- a/src/meshdb/utils/spreadsheet_import/main.py +++ b/src/meshdb/utils/spreadsheet_import/main.py @@ -3,7 +3,7 @@ import sys import time from collections import defaultdict -from typing import List +from typing import Dict, List import django @@ -16,6 +16,7 @@ django.setup() from meshapi import models +from meshdb.utils.spreadsheet_import.building.constants import INVALID_BIN_NUMBERS from meshdb.utils.spreadsheet_import.building.resolve_address import AddressParser from meshdb.utils.spreadsheet_import.csv_load import ( DroppedModification, @@ -29,7 +30,7 @@ from meshdb.utils.spreadsheet_import.parse_install import create_install, normalize_install_to_primary_building_node from meshdb.utils.spreadsheet_import.parse_link import load_links_supplement_with_uisp from meshdb.utils.spreadsheet_import.parse_member import get_or_create_member -from meshdb.utils.spreadsheet_import.parse_node import get_or_create_node, normalize_building_node_links +from meshdb.utils.spreadsheet_import.parse_node import get_node_type, get_or_create_node, normalize_building_node_links def main(): @@ -51,20 +52,42 @@ def main(): form_responses_path, links_path, sectors_path = sys.argv[1:4] - rows, skipped = get_spreadsheet_rows(form_responses_path) + rows, reassigned_rows, skipped = get_spreadsheet_rows(form_responses_path) logging.info(f'Loaded {len(rows)} rows from "{form_responses_path}"') - member_duplicate_counts = defaultdict(lambda: 1) - - addr_parser = AddressParser() - - dropped_modifications: List[DroppedModification] = [] - - max_install_num = max(row.id for row in rows) - - start_time = time.time() - logging.info(f"Processing install # {rows[0].id}/{max_install_num}...") try: + logging.info(f"Creating {len(reassigned_rows)} nodes for rows marked 'NN Reassigned'...") + + nn_bin_map: Dict[int, int] = {} + for row in reassigned_rows: + node = models.Node( + network_number=row.id, + name=row.nodeName if row.nodeName else None, + latitude=row.latitude, + longitude=row.longitude, + altitude=row.altitude, + status=models.Node.NodeStatus.PLANNED, # This will get overridden later + type=get_node_type(row.notes) if row.notes else models.Node.NodeType.STANDARD, + notes=f"Spreadsheet Notes:\n" + f"{row.notes if row.notes else None}\n\n" + f"Spreadsheet Notes2:\n" + f"{row.notes2 if row.notes2 else None}\n\n", + ) + node.save() + dob_bin = row.bin if row.bin and row.bin > 0 and row.bin not in INVALID_BIN_NUMBERS else None + if dob_bin: + nn_bin_map[node.network_number] = dob_bin + + member_duplicate_counts = defaultdict(lambda: 1) + + addr_parser = AddressParser() + + dropped_modifications: List[DroppedModification] = [] + + max_install_num = max(row.id for row in rows) + + start_time = time.time() + logging.info(f"Processing install # {rows[0].id}/{max_install_num}...") for i, row in enumerate(rows): if (i + 2) % 100 == 0: logging.info( @@ -146,6 +169,34 @@ def main(): for install in models.Install.objects.all(): normalize_install_to_primary_building_node(install) + # Confirm that the appropriate NN -> Building relations have been formed via the Install + # import that we would expect from the NN only rows + for nn, _bin in nn_bin_map.items(): + node = models.Node.objects.get(network_number=nn) + building_match = node.buildings.filter(bin=_bin) + if not building_match: + logging.warning( + f"Warning, from NN data, expected NN{nn} to be connected to at least one building " + f"with DOB number {_bin} but no such connection was found. Adding it now..." + ) + building_candidates = models.Building.objects.filter(bin=_bin) + if len(building_candidates) == 0: + logging.error( + f"Found no buildings with DOB BIN {_bin}, but this BIN is specified in " + f"spreadsheet row #{nn}. Is this BIN correct?" + ) + continue + for building in building_candidates: + node.buildings.add(building) + + for node in models.Node.objects.all(): + if not node.installs.all(): + # If we don't have any installs associated with this node, it is not + # active or planned, mark it as INACTIVE + logging.warning(f"Found node imported without installs (NN{node.network_number}), marking INACTIVE") + node.status = models.Node.NodeStatus.INACTIVE + node.save() + # Create an AP device for each access point install load_access_points(rows) diff --git a/src/meshdb/utils/spreadsheet_import/parse_building.py b/src/meshdb/utils/spreadsheet_import/parse_building.py index 493ff8d1..ace6b15e 100644 --- a/src/meshdb/utils/spreadsheet_import/parse_building.py +++ b/src/meshdb/utils/spreadsheet_import/parse_building.py @@ -53,6 +53,7 @@ def get_existing_building( def diff_new_building_against_existing( row_id: int, + row_status: str, existing_building: models.Building, new_building: models.Building, add_dropped_edit: Callable[[DroppedModification], None], @@ -63,6 +64,7 @@ def diff_new_building_against_existing( DroppedModification( list(install.install_number for install in existing_building.installs.all()), row_id, + row_status, existing_building.street_address, "building.bin", str(existing_building.bin) if existing_building.bin else "", @@ -80,6 +82,7 @@ def diff_new_building_against_existing( DroppedModification( list(install.install_number for install in existing_building.installs.all()), row_id, + row_status, str(existing_building.bin) if existing_building.bin else existing_building.street_address, "building.street_address", existing_building.street_address if existing_building.street_address else "", @@ -97,6 +100,7 @@ def diff_new_building_against_existing( DroppedModification( list(install.install_number for install in existing_building.installs.all()), row_id, + row_status, str(existing_building.bin) if existing_building.bin else existing_building.street_address, "building.city", existing_building.city if existing_building.city else "", @@ -114,6 +118,7 @@ def diff_new_building_against_existing( DroppedModification( list(install.install_number for install in existing_building.installs.all()), row_id, + row_status, str(existing_building.bin) if existing_building.bin else existing_building.street_address, "building.state", existing_building.state if existing_building.state else "", @@ -131,6 +136,7 @@ def diff_new_building_against_existing( DroppedModification( list(install.install_number for install in existing_building.installs.all()), row_id, + row_status, str(existing_building.bin) if existing_building.bin else existing_building.street_address, "building.zip_code", existing_building.zip_code if existing_building.zip_code else "", @@ -178,14 +184,11 @@ def nop(*args, **kwargs): DroppedModification( [row.id], row.id, - ( - address_result.discovered_bin - if address_result.discovered_bin - else address_result.address.street_address - ), + row.status.value, + row.address, "lat_long_discrepancy_vs_spreadsheet", - str(address_result.discovered_lat_lon), str((row.latitude, row.longitude)), + str(address_result.discovered_lat_lon), ) ) distance_warning = ( @@ -214,6 +217,7 @@ def nop(*args, **kwargs): if existing_building: diff_notes = diff_new_building_against_existing( row.id, + row.status.value, existing_building, models.Building( bin=address_result.discovered_bin or dob_bin, diff --git a/src/meshdb/utils/spreadsheet_import/parse_member.py b/src/meshdb/utils/spreadsheet_import/parse_member.py index c679a1b9..ed0739f3 100644 --- a/src/meshdb/utils/spreadsheet_import/parse_member.py +++ b/src/meshdb/utils/spreadsheet_import/parse_member.py @@ -100,6 +100,7 @@ def parse_phone(input_phone: str) -> Optional[phonenumbers.PhoneNumber]: def diff_new_member_against_existing( row_id: int, + row_status: str, existing_member: models.Member, new_member: models.Member, add_dropped_edit: Callable[[DroppedModification], None], @@ -110,6 +111,7 @@ def diff_new_member_against_existing( DroppedModification( list(install.install_number for install in existing_member.installs.all()), row_id, + row_status, existing_member.primary_email_address, "member.name", existing_member.name if existing_member.name else "", @@ -130,6 +132,7 @@ def diff_new_member_against_existing( DroppedModification( list(install.install_number for install in existing_member.installs.all()), row_id, + row_status, existing_member.primary_email_address, "member.phone_number", existing_member.phone_number, @@ -224,6 +227,7 @@ def nop(*args, **kwargs): diff_notes = diff_new_member_against_existing( row.id, + row.status.value, existing_members[0], models.Member( name=row.name, diff --git a/src/meshdb/utils/spreadsheet_import/parse_node.py b/src/meshdb/utils/spreadsheet_import/parse_node.py index edc385f1..de8316e1 100644 --- a/src/meshdb/utils/spreadsheet_import/parse_node.py +++ b/src/meshdb/utils/spreadsheet_import/parse_node.py @@ -43,6 +43,9 @@ def get_or_create_node( if len(existing_nodes): node = existing_nodes[0] + if not node.name and row.nodeName: + node.name = row.nodeName + if not node.install_date or (row.installDate and row.installDate < node.install_date): node.install_date = row.installDate