Partial import from NN re-assigned rows instead of dropping them (#333)

* Partial import from NN re-assigned rows instead of dropping them

* Mark nodes without associated installs inactive

* Improve diagnostic info for distance based dropped edit reports

* Add row status column to dropped edit report

src/meshdb/utils/spreadsheet_import/csv_load.py

@@ -101,6 +101,7 @@ class SpreadsheetSector:
 class DroppedModification:
     original_row_ids: List[int]
     new_row_id: int
+    row_status: str
     deduplication_value: str
     modified_property: str
     database_value: str
@@ -109,12 +110,13 @@ class DroppedModification:
 
 def get_spreadsheet_rows(
     form_responses_path: str,
-) -> Tuple[List[SpreadsheetRow], Dict[int, str]]:
+) -> Tuple[List[SpreadsheetRow], List[SpreadsheetRow], Dict[int, str]]:
     with open(form_responses_path, "r") as input_file:
         csv_reader = csv.DictReader(input_file)
 
         skipped_rows: Dict[int, str] = {}
-        nodes: List[SpreadsheetRow] = []
+        installs: List[SpreadsheetRow] = []
+        reassigned_nns: List[SpreadsheetRow] = []
 
         for i, row in enumerate(csv_reader):
             # Last row is placeholder
@@ -140,11 +142,16 @@ def get_spreadsheet_rows(
             except ValueError:
                 abandon_date = None
 
+            node_status = SpreadsheetStatus(row["Status"].replace("dupe", "Dupe"))
+
             re_assigned_as_nn = False
             try:
                 nn = row["NN"].lower().strip()
-                re_assigned_as_nn = nn.startswith("x-")
-                nn = int(nn) if nn is not None and nn != "" and not re_assigned_as_nn else None
+                re_assigned_as_nn = nn.startswith("x-") or node_status == SpreadsheetStatus.nnAssigned
+                if re_assigned_as_nn:
+                    nn = node_id
+                else:
+                    nn = int(nn) if nn is not None and nn != "" else None
             except (KeyError, ValueError):
                 nn = None
 
@@ -159,7 +166,7 @@ def get_spreadsheet_rows(
                 secondEmail=row["2nd profile email"].lower().strip(),
                 phone=row["Phone"],
                 roofAccess=row["Rooftop Access"] == "I have Rooftop access",
-                status=SpreadsheetStatus(row["Status"].replace("dupe", "Dupe")),
+                status=node_status,
                 installDate=install_date,
                 abandonDate=abandon_date,
                 nodeName=row["nodeName"],
@@ -181,12 +188,12 @@ def get_spreadsheet_rows(
                 continue
 
             if re_assigned_as_nn:
-                skipped_rows[node_id] = "Reassigned as NN for another row"
+                reassigned_nns.append(node)
                 continue
 
-            nodes.append(node)
+            installs.append(node)
 
-        return nodes, skipped_rows
+        return installs, reassigned_nns, skipped_rows
 
 
 def print_failure_report(skipped_rows: Dict[int, str], original_input_file: str, fname_overide: str = None) -> None:
@@ -234,6 +241,7 @@ def print_dropped_edit_report(
                 [
                     "OriginalRowID(s)",
                     "DroppedRowID",
+                    "DroppedRowSpreadsheetStatus",
                     "DeduplicationValue",
                     "ModifiedProperty",
                     "DatabaseValue",
@@ -249,6 +257,7 @@ def print_dropped_edit_report(
                         new_fields = {}
                         new_fields["OriginalRowID(s)"] = ", ".join(str(row_id) for row_id in edit.original_row_ids)
                         new_fields["DroppedRowID"] = edit.new_row_id
+                        new_fields["DroppedRowSpreadsheetStatus"] = edit.row_status
                         new_fields["DeduplicationValue"] = edit.deduplication_value
                         new_fields["ModifiedProperty"] = edit.modified_property
                         new_fields["DatabaseValue"] = edit.database_value

src/meshdb/utils/spreadsheet_import/main.py

@@ -3,7 +3,7 @@
 import sys
 import time
 from collections import defaultdict
-from typing import List
+from typing import Dict, List
 
 import django
 
@@ -16,6 +16,7 @@
 django.setup()
 
 from meshapi import models
+from meshdb.utils.spreadsheet_import.building.constants import INVALID_BIN_NUMBERS
 from meshdb.utils.spreadsheet_import.building.resolve_address import AddressParser
 from meshdb.utils.spreadsheet_import.csv_load import (
     DroppedModification,
@@ -29,7 +30,7 @@
 from meshdb.utils.spreadsheet_import.parse_install import create_install, normalize_install_to_primary_building_node
 from meshdb.utils.spreadsheet_import.parse_link import load_links_supplement_with_uisp
 from meshdb.utils.spreadsheet_import.parse_member import get_or_create_member
-from meshdb.utils.spreadsheet_import.parse_node import get_or_create_node, normalize_building_node_links
+from meshdb.utils.spreadsheet_import.parse_node import get_node_type, get_or_create_node, normalize_building_node_links
 
 
 def main():
@@ -51,20 +52,42 @@ def main():
 
     form_responses_path, links_path, sectors_path = sys.argv[1:4]
 
-    rows, skipped = get_spreadsheet_rows(form_responses_path)
+    rows, reassigned_rows, skipped = get_spreadsheet_rows(form_responses_path)
     logging.info(f'Loaded {len(rows)} rows from "{form_responses_path}"')
 
-    member_duplicate_counts = defaultdict(lambda: 1)
-
-    addr_parser = AddressParser()
-
-    dropped_modifications: List[DroppedModification] = []
-
-    max_install_num = max(row.id for row in rows)
-
-    start_time = time.time()
-    logging.info(f"Processing install # {rows[0].id}/{max_install_num}...")
     try:
+        logging.info(f"Creating {len(reassigned_rows)} nodes for rows marked 'NN Reassigned'...")
+
+        nn_bin_map: Dict[int, int] = {}
+        for row in reassigned_rows:
+            node = models.Node(
+                network_number=row.id,
+                name=row.nodeName if row.nodeName else None,
+                latitude=row.latitude,
+                longitude=row.longitude,
+                altitude=row.altitude,
+                status=models.Node.NodeStatus.PLANNED,  # This will get overridden later
+                type=get_node_type(row.notes) if row.notes else models.Node.NodeType.STANDARD,
+                notes=f"Spreadsheet Notes:\n"
+                f"{row.notes if row.notes else None}\n\n"
+                f"Spreadsheet Notes2:\n"
+                f"{row.notes2 if row.notes2 else None}\n\n",
+            )
+            node.save()
+            dob_bin = row.bin if row.bin and row.bin > 0 and row.bin not in INVALID_BIN_NUMBERS else None
+            if dob_bin:
+                nn_bin_map[node.network_number] = dob_bin
+
+        member_duplicate_counts = defaultdict(lambda: 1)
+
+        addr_parser = AddressParser()
+
+        dropped_modifications: List[DroppedModification] = []
+
+        max_install_num = max(row.id for row in rows)
+
+        start_time = time.time()
+        logging.info(f"Processing install # {rows[0].id}/{max_install_num}...")
         for i, row in enumerate(rows):
             if (i + 2) % 100 == 0:
                 logging.info(
@@ -146,6 +169,34 @@ def main():
         for install in models.Install.objects.all():
             normalize_install_to_primary_building_node(install)
 
+        # Confirm that the appropriate NN -> Building relations have been formed via the Install
+        # import that we would expect from the NN only rows
+        for nn, _bin in nn_bin_map.items():
+            node = models.Node.objects.get(network_number=nn)
+            building_match = node.buildings.filter(bin=_bin)
+            if not building_match:
+                logging.warning(
+                    f"Warning, from NN data, expected NN{nn} to be connected to at least one building "
+                    f"with DOB number {_bin} but no such connection was found. Adding it now..."
+                )
+                building_candidates = models.Building.objects.filter(bin=_bin)
+                if len(building_candidates) == 0:
+                    logging.error(
+                        f"Found no buildings with DOB BIN {_bin}, but this BIN is specified in "
+                        f"spreadsheet row #{nn}. Is this BIN correct?"
+                    )
+                    continue
+                for building in building_candidates:
+                    node.buildings.add(building)
+
+        for node in models.Node.objects.all():
+            if not node.installs.all():
+                # If we don't have any installs associated with this node, it is not
+                # active or planned, mark it as INACTIVE
+                logging.warning(f"Found node imported without installs (NN{node.network_number}), marking INACTIVE")
+                node.status = models.Node.NodeStatus.INACTIVE
+                node.save()
+
         # Create an AP device for each access point install
         load_access_points(rows)
 

src/meshdb/utils/spreadsheet_import/parse_building.py

@@ -53,6 +53,7 @@ def get_existing_building(
 
 def diff_new_building_against_existing(
     row_id: int,
+    row_status: str,
     existing_building: models.Building,
     new_building: models.Building,
     add_dropped_edit: Callable[[DroppedModification], None],
@@ -63,6 +64,7 @@ def diff_new_building_against_existing(
             DroppedModification(
                 list(install.install_number for install in existing_building.installs.all()),
                 row_id,
+                row_status,
                 existing_building.street_address,
                 "building.bin",
                 str(existing_building.bin) if existing_building.bin else "",
@@ -80,6 +82,7 @@ def diff_new_building_against_existing(
             DroppedModification(
                 list(install.install_number for install in existing_building.installs.all()),
                 row_id,
+                row_status,
                 str(existing_building.bin) if existing_building.bin else existing_building.street_address,
                 "building.street_address",
                 existing_building.street_address if existing_building.street_address else "",
@@ -97,6 +100,7 @@ def diff_new_building_against_existing(
             DroppedModification(
                 list(install.install_number for install in existing_building.installs.all()),
                 row_id,
+                row_status,
                 str(existing_building.bin) if existing_building.bin else existing_building.street_address,
                 "building.city",
                 existing_building.city if existing_building.city else "",
@@ -114,6 +118,7 @@ def diff_new_building_against_existing(
             DroppedModification(
                 list(install.install_number for install in existing_building.installs.all()),
                 row_id,
+                row_status,
                 str(existing_building.bin) if existing_building.bin else existing_building.street_address,
                 "building.state",
                 existing_building.state if existing_building.state else "",
@@ -131,6 +136,7 @@ def diff_new_building_against_existing(
             DroppedModification(
                 list(install.install_number for install in existing_building.installs.all()),
                 row_id,
+                row_status,
                 str(existing_building.bin) if existing_building.bin else existing_building.street_address,
                 "building.zip_code",
                 existing_building.zip_code if existing_building.zip_code else "",
@@ -178,14 +184,11 @@ def nop(*args, **kwargs):
                 DroppedModification(
                     [row.id],
                     row.id,
-                    (
-                        address_result.discovered_bin
-                        if address_result.discovered_bin
-                        else address_result.address.street_address
-                    ),
+                    row.status.value,
+                    row.address,
                     "lat_long_discrepancy_vs_spreadsheet",
-                    str(address_result.discovered_lat_lon),
                     str((row.latitude, row.longitude)),
+                    str(address_result.discovered_lat_lon),
                 )
             )
             distance_warning = (
@@ -214,6 +217,7 @@ def nop(*args, **kwargs):
     if existing_building:
         diff_notes = diff_new_building_against_existing(
             row.id,
+            row.status.value,
             existing_building,
             models.Building(
                 bin=address_result.discovered_bin or dob_bin,

src/meshdb/utils/spreadsheet_import/parse_member.py

@@ -100,6 +100,7 @@ def parse_phone(input_phone: str) -> Optional[phonenumbers.PhoneNumber]:
 
 def diff_new_member_against_existing(
     row_id: int,
+    row_status: str,
     existing_member: models.Member,
     new_member: models.Member,
     add_dropped_edit: Callable[[DroppedModification], None],
@@ -110,6 +111,7 @@ def diff_new_member_against_existing(
             DroppedModification(
                 list(install.install_number for install in existing_member.installs.all()),
                 row_id,
+                row_status,
                 existing_member.primary_email_address,
                 "member.name",
                 existing_member.name if existing_member.name else "",
@@ -130,6 +132,7 @@ def diff_new_member_against_existing(
             DroppedModification(
                 list(install.install_number for install in existing_member.installs.all()),
                 row_id,
+                row_status,
                 existing_member.primary_email_address,
                 "member.phone_number",
                 existing_member.phone_number,
@@ -224,6 +227,7 @@ def nop(*args, **kwargs):
 
             diff_notes = diff_new_member_against_existing(
                 row.id,
+                row.status.value,
                 existing_members[0],
                 models.Member(
                     name=row.name,

src/meshdb/utils/spreadsheet_import/parse_node.py

@@ -43,6 +43,9 @@ def get_or_create_node(
     if len(existing_nodes):
         node = existing_nodes[0]
 
+        if not node.name and row.nodeName:
+            node.name = row.nodeName
+
         if not node.install_date or (row.installDate and row.installDate < node.install_date):
             node.install_date = row.installDate