Load Ticket Number from OSTicket during import process (#517)

* Rename ticket ID -> ticket number

* Add Install ticket number import from OSTicket

* Merge migrations

* ticket_id -> ticket_number in serializer test

src/meshapi/admin/models/install.py

@@ -35,8 +35,8 @@ class Meta:
         fields = "__all__"
         widgets = {
             "unit": forms.TextInput(),
-            "ticket_id": ExternalHyperlinkWidget(
-                lambda ticket_id: f"{OSTICKET_URL}/scp/tickets.php?id={ticket_id}",
+            "ticket_number": ExternalHyperlinkWidget(
+                lambda ticket_number: f"{OSTICKET_URL}/scp/tickets.php?number={ticket_number}",
                 title="View in OSTicket",
             ),
         }
@@ -92,7 +92,7 @@ class InstallAdmin(RankedSearchMixin, ImportExportModelAdmin, ExportActionMixin)
                 "fields": [
                     "install_number",
                     "status",
-                    "ticket_id",
+                    "ticket_number",
                     "member",
                 ]
             },

src/meshapi/migrations/0017_rename_ticket_number.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.9 on 2024-09-08 23:45
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("meshapi", "0016_alter_los_analysis_date"),
+    ]
+
+    operations = [
+        migrations.RenameField(
+            model_name="install",
+            old_name="ticket_id",
+            new_name="ticket_number",
+        ),
+        migrations.AlterField(
+            model_name="install",
+            name="ticket_number",
+            field=models.IntegerField(
+                blank=True,
+                help_text="The ticket number of the OSTicket used to track communications with the member about this install",
+                null=True,
+            ),
+        ),
+    ]

src/meshapi/migrations/0029_merge_20240916_1919.py

@@ -0,0 +1,12 @@
+# Generated by Django 4.2.13 on 2024-09-16 23:19
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("meshapi", "0017_rename_ticket_number"),
+        ("meshapi", "0028_merge_0017_permission_0027_doc_changes"),
+    ]
+
+    operations = []

src/meshapi/models/install.py

@@ -52,11 +52,11 @@ class InstallStatus(models.TextChoices):
         help_text="The current status of this install",
     )
 
-    # OSTicket ID
-    ticket_id = models.IntegerField(
+    # OSTicket Ticket Number
+    ticket_number = models.IntegerField(
         blank=True,
         null=True,
-        help_text="The ID of the OSTicket used to track communications with the member about this install",
+        help_text="The ticket number of the OSTicket used to track communications with the member about this install",
     )
 
     # Important dates

src/meshapi/tests/sample_data.py

@@ -33,7 +33,7 @@
 
 sample_install = {
     "status": Install.InstallStatus.ACTIVE,
-    "ticket_id": 69,
+    "ticket_number": 69,
     "request_date": "2022-02-27",
     "install_date": "2022-03-01",
     "abandon_date": "9999-01-01",

src/meshapi/tests/test_nn.py

@@ -637,7 +637,7 @@ def add_data(self, b, m, i, index=101, create_node=False):
         m["primary_email_address"] = f"john{index}@gmail.com"
         member_obj = Member(**m)
         i["member"] = member_obj
-        i["ticket_id"] = index
+        i["ticket_number"] = index
         install_obj = Install(**i, install_number=index)
 
         if create_node:

src/meshapi/tests/test_serializers.py

@@ -176,7 +176,7 @@ def test_views_get_install(self):
             "request_date": "2022-02-27",
             "roof_access": True,
             "status": "Active",
-            "ticket_id": 69,
+            "ticket_number": 69,
             "unit": "3",
         }
         response = self._call(f"/api/v1/installs/{self.install.id}/", 200)

src/meshapi/views/forms.py

@@ -223,7 +223,7 @@ def join_form(request: Request) -> Response:
 
     join_form_install = Install(
         status=Install.InstallStatus.REQUEST_RECEIVED,
-        ticket_id=None,
+        ticket_number=None,
         request_date=date.today(),
         install_date=None,
         abandon_date=None,

src/meshdb/utils/spreadsheet_import/fetch_osticket_data.py

@@ -0,0 +1,148 @@
+import csv
+import logging
+import os
+import sys
+import time
+from io import StringIO
+
+import bs4
+import django
+import requests
+
+from meshdb.utils.spreadsheet_import import logger
+
+logger.configure()
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshdb.settings")
+django.setup()
+
+from meshapi.models import Install
+
+
+def download_export(session, export_id):
+    download_response = session.get("https://support.nycmesh.net/scp/export.php?id=" + export_id)
+
+    if download_response.status_code == 416:
+        return None
+
+    if download_response.status_code != 200:
+        download_response.raise_for_status()
+
+    logging.info("Downloading OSTicket Export....")
+    return download_response.text
+
+
+def attempt_to_load_osticket_data(queue_number):
+    username = os.environ.get("OSTICKET_USER")
+    password = os.environ.get("OSTICKET_PASSWORD")
+
+    if not username or not password:
+        raise EnvironmentError("OSTICKET_USER and OSTICKET_PASSWORD env vars must be set")
+
+    logging.info(f"Authenticating to OSTicket (queue {queue_number})....")
+
+    session = requests.Session()
+    response_for_csrf = session.get("https://support.nycmesh.net")
+    headers = {"Content-Type": "application/x-www-form-urlencoded"}
+
+    soup = bs4.BeautifulSoup(response_for_csrf.text, "html.parser")
+    csrf_token = soup.find("meta", attrs={"name": "csrf_token"}).get("content")
+
+    assert csrf_token
+
+    login_response = session.post(
+        "https://support.nycmesh.net/scp/login.php",
+        data={
+            "__CSRFToken__": csrf_token,
+            "do": "scplogin",
+            "userid": "[email protected]",
+            "passwd": "kFydO9WJDLonP3wG",
+            "ajax": "1",
+        },
+        headers=headers,
+    )
+    assert login_response.json() == {"status": 302, "redirect": "index.php"}
+
+    logging.info(f"Requestng OSTicket Export (queue {queue_number})....")
+    export_response = session.post(
+        f"https://support.nycmesh.net/scp/ajax.php/tickets/export/{queue_number}",
+        headers=headers,
+        data=[
+            ("fields[]", "number"),
+            ("fields[]", "cdata__subject"),
+            ("fields[]", "topic_id"),
+            ("fields[]", "cdata__node"),
+            ("fields[]", "cdata__rooftop"),
+            ("filename", "Closed Tickets - ABC.csv"),
+            ("csv-delimiter", ","),
+            ("undefined", "Export"),
+            ("__CSRFToken__", csrf_token),
+        ],
+    )
+    eid = export_response.json()["eid"]
+
+    logging.info(f"Waiting for OSTicket Export to complete (queue {queue_number})....")
+    csv_contents = download_export(session, eid)
+    attempts = 1
+    while csv_contents is None:
+        csv_contents = download_export(session, eid)
+        attempts += 1
+        time.sleep(5)
+
+        if attempts > 20:
+            raise TimeoutError("Too many attempts to download export data")
+
+    f = StringIO(csv_contents)
+    reader = csv.DictReader(f)
+    return list(reader)
+
+
+def parse_node_text_to_install_number(node_text):
+    if not node_text:
+        return None
+
+    try:
+        return int(node_text)
+    except ValueError:
+        modified_text = node_text.strip().replace("#", "").split(" ")[0]
+        try:
+            return int(modified_text)
+        except ValueError:
+            logging.error(f"Bad node: {node_text}")
+            return None
+
+
+def import_ticket_numbers_from_osticket():
+    attempts = 0
+    while True:
+        try:
+            attempts += 1
+            closed_tickets = attempt_to_load_osticket_data("8")
+            break
+        except Exception as e:
+            if attempts > 3:
+                raise e
+
+    while True:
+        try:
+            attempts += 1
+            open_tickets = attempt_to_load_osticket_data("1")
+            break
+        except Exception as e:
+            if attempts > 3:
+                raise e
+
+    logging.info("Loading ticket numbers into install objects...")
+    for ticket in open_tickets + closed_tickets:
+        if ticket["node"]:
+            install_number = parse_node_text_to_install_number(ticket["node"])
+            if install_number:
+                ticket_number = ticket["Ticket Number"]
+                install: Install = Install.objects.filter(install_number=install_number).first()
+                if install:
+                    install.ticket_number = ticket_number
+                    install.save()
+
+
+if __name__ == "__main__":
+    import_ticket_numbers_from_osticket()

src/meshdb/utils/spreadsheet_import/main.py

@@ -8,6 +8,7 @@
 import django
 
 from meshdb.utils.spreadsheet_import import logger
+from meshdb.utils.spreadsheet_import.fetch_osticket_data import import_ticket_numbers_from_osticket
 
 logger.configure()
 
@@ -228,6 +229,9 @@ def main():
 
         logging.info(f"Importing links from UISP & '{links_path}'")
         load_links_supplement_with_uisp(get_spreadsheet_links((links_path)))
+
+        logging.info(f"Importing ticket numbers from OSTicket")
+        import_ticket_numbers_from_osticket()
     except BaseException as e:
         if isinstance(e, KeyboardInterrupt):
             logging.error("Received keyboard interrupt, exiting early...")

src/meshdb/utils/spreadsheet_import/parse_install.py

@@ -44,9 +44,7 @@ def create_install(row: SpreadsheetRow) -> Optional[models.Install]:
     install = models.Install(
         install_number=row.id,
         status=translate_spreadsheet_status_to_db_status(row.status),
-        ticket_id=None,
-        # TODO: Figure out if we can export data from OSTicket to back-fill this
-        #  https://github.com/nycmeshnet/meshdb/issues/510
+        ticket_number=None,
         request_date=row.request_date.date(),
         install_date=row.installDate,
         abandon_date=row.abandonDate,