From ed1ee38e8443b8f8a18eb74b84b95ee32cb98118 Mon Sep 17 00:00:00 2001 From: Andrew Dickinson Date: Mon, 16 Sep 2024 19:35:50 -0400 Subject: [PATCH] Load Ticket Number from OSTicket during import process (#517) * Rename ticket ID -> ticket number * Add Install ticket number import from OSTicket * Merge migrations * ticket_id -> ticket_number in serializer test --- src/meshapi/admin/models/install.py | 6 +- .../migrations/0017_rename_ticket_number.py | 26 +++ .../migrations/0029_merge_20240916_1919.py | 12 ++ src/meshapi/models/install.py | 6 +- src/meshapi/tests/sample_data.py | 2 +- src/meshapi/tests/test_nn.py | 2 +- src/meshapi/tests/test_serializers.py | 2 +- src/meshapi/views/forms.py | 2 +- .../spreadsheet_import/fetch_osticket_data.py | 148 ++++++++++++++++++ src/meshdb/utils/spreadsheet_import/main.py | 4 + .../utils/spreadsheet_import/parse_install.py | 4 +- 11 files changed, 201 insertions(+), 13 deletions(-) create mode 100644 src/meshapi/migrations/0017_rename_ticket_number.py create mode 100644 src/meshapi/migrations/0029_merge_20240916_1919.py create mode 100644 src/meshdb/utils/spreadsheet_import/fetch_osticket_data.py diff --git a/src/meshapi/admin/models/install.py b/src/meshapi/admin/models/install.py index 45561858..821dca53 100644 --- a/src/meshapi/admin/models/install.py +++ b/src/meshapi/admin/models/install.py @@ -35,8 +35,8 @@ class Meta: fields = "__all__" widgets = { "unit": forms.TextInput(), - "ticket_id": ExternalHyperlinkWidget( - lambda ticket_id: f"{OSTICKET_URL}/scp/tickets.php?id={ticket_id}", + "ticket_number": ExternalHyperlinkWidget( + lambda ticket_number: f"{OSTICKET_URL}/scp/tickets.php?number={ticket_number}", title="View in OSTicket", ), } @@ -92,7 +92,7 @@ class InstallAdmin(RankedSearchMixin, ImportExportModelAdmin, ExportActionMixin) "fields": [ "install_number", "status", - "ticket_id", + "ticket_number", "member", ] }, diff --git a/src/meshapi/migrations/0017_rename_ticket_number.py b/src/meshapi/migrations/0017_rename_ticket_number.py new file mode 100644 index 00000000..0a889d86 --- /dev/null +++ b/src/meshapi/migrations/0017_rename_ticket_number.py @@ -0,0 +1,26 @@ +# Generated by Django 4.2.9 on 2024-09-08 23:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("meshapi", "0016_alter_los_analysis_date"), + ] + + operations = [ + migrations.RenameField( + model_name="install", + old_name="ticket_id", + new_name="ticket_number", + ), + migrations.AlterField( + model_name="install", + name="ticket_number", + field=models.IntegerField( + blank=True, + help_text="The ticket number of the OSTicket used to track communications with the member about this install", + null=True, + ), + ), + ] diff --git a/src/meshapi/migrations/0029_merge_20240916_1919.py b/src/meshapi/migrations/0029_merge_20240916_1919.py new file mode 100644 index 00000000..3fbd7af2 --- /dev/null +++ b/src/meshapi/migrations/0029_merge_20240916_1919.py @@ -0,0 +1,12 @@ +# Generated by Django 4.2.13 on 2024-09-16 23:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + dependencies = [ + ("meshapi", "0017_rename_ticket_number"), + ("meshapi", "0028_merge_0017_permission_0027_doc_changes"), + ] + + operations = [] diff --git a/src/meshapi/models/install.py b/src/meshapi/models/install.py index 928ceff1..5fad48c6 100644 --- a/src/meshapi/models/install.py +++ b/src/meshapi/models/install.py @@ -52,11 +52,11 @@ class InstallStatus(models.TextChoices): help_text="The current status of this install", ) - # OSTicket ID - ticket_id = models.IntegerField( + # OSTicket Ticket Number + ticket_number = models.IntegerField( blank=True, null=True, - help_text="The ID of the OSTicket used to track communications with the member about this install", + help_text="The ticket number of the OSTicket used to track communications with the member about this install", ) # Important dates diff --git a/src/meshapi/tests/sample_data.py b/src/meshapi/tests/sample_data.py index 06373a89..b094f12d 100644 --- a/src/meshapi/tests/sample_data.py +++ b/src/meshapi/tests/sample_data.py @@ -33,7 +33,7 @@ sample_install = { "status": Install.InstallStatus.ACTIVE, - "ticket_id": 69, + "ticket_number": 69, "request_date": "2022-02-27", "install_date": "2022-03-01", "abandon_date": "9999-01-01", diff --git a/src/meshapi/tests/test_nn.py b/src/meshapi/tests/test_nn.py index 3106f1c7..f02aa894 100644 --- a/src/meshapi/tests/test_nn.py +++ b/src/meshapi/tests/test_nn.py @@ -637,7 +637,7 @@ def add_data(self, b, m, i, index=101, create_node=False): m["primary_email_address"] = f"john{index}@gmail.com" member_obj = Member(**m) i["member"] = member_obj - i["ticket_id"] = index + i["ticket_number"] = index install_obj = Install(**i, install_number=index) if create_node: diff --git a/src/meshapi/tests/test_serializers.py b/src/meshapi/tests/test_serializers.py index 934ac850..aa110be1 100644 --- a/src/meshapi/tests/test_serializers.py +++ b/src/meshapi/tests/test_serializers.py @@ -176,7 +176,7 @@ def test_views_get_install(self): "request_date": "2022-02-27", "roof_access": True, "status": "Active", - "ticket_id": 69, + "ticket_number": 69, "unit": "3", } response = self._call(f"/api/v1/installs/{self.install.id}/", 200) diff --git a/src/meshapi/views/forms.py b/src/meshapi/views/forms.py index 1e70b7ef..b9483aa3 100644 --- a/src/meshapi/views/forms.py +++ b/src/meshapi/views/forms.py @@ -223,7 +223,7 @@ def join_form(request: Request) -> Response: join_form_install = Install( status=Install.InstallStatus.REQUEST_RECEIVED, - ticket_id=None, + ticket_number=None, request_date=date.today(), install_date=None, abandon_date=None, diff --git a/src/meshdb/utils/spreadsheet_import/fetch_osticket_data.py b/src/meshdb/utils/spreadsheet_import/fetch_osticket_data.py new file mode 100644 index 00000000..d1322c79 --- /dev/null +++ b/src/meshdb/utils/spreadsheet_import/fetch_osticket_data.py @@ -0,0 +1,148 @@ +import csv +import logging +import os +import sys +import time +from io import StringIO + +import bs4 +import django +import requests + +from meshdb.utils.spreadsheet_import import logger + +logger.configure() + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "meshdb.settings") +django.setup() + +from meshapi.models import Install + + +def download_export(session, export_id): + download_response = session.get("https://support.nycmesh.net/scp/export.php?id=" + export_id) + + if download_response.status_code == 416: + return None + + if download_response.status_code != 200: + download_response.raise_for_status() + + logging.info("Downloading OSTicket Export....") + return download_response.text + + +def attempt_to_load_osticket_data(queue_number): + username = os.environ.get("OSTICKET_USER") + password = os.environ.get("OSTICKET_PASSWORD") + + if not username or not password: + raise EnvironmentError("OSTICKET_USER and OSTICKET_PASSWORD env vars must be set") + + logging.info(f"Authenticating to OSTicket (queue {queue_number})....") + + session = requests.Session() + response_for_csrf = session.get("https://support.nycmesh.net") + headers = {"Content-Type": "application/x-www-form-urlencoded"} + + soup = bs4.BeautifulSoup(response_for_csrf.text, "html.parser") + csrf_token = soup.find("meta", attrs={"name": "csrf_token"}).get("content") + + assert csrf_token + + login_response = session.post( + "https://support.nycmesh.net/scp/login.php", + data={ + "__CSRFToken__": csrf_token, + "do": "scplogin", + "userid": "andrew.dickinson.0216@gmail.com", + "passwd": "kFydO9WJDLonP3wG", + "ajax": "1", + }, + headers=headers, + ) + assert login_response.json() == {"status": 302, "redirect": "index.php"} + + logging.info(f"Requestng OSTicket Export (queue {queue_number})....") + export_response = session.post( + f"https://support.nycmesh.net/scp/ajax.php/tickets/export/{queue_number}", + headers=headers, + data=[ + ("fields[]", "number"), + ("fields[]", "cdata__subject"), + ("fields[]", "topic_id"), + ("fields[]", "cdata__node"), + ("fields[]", "cdata__rooftop"), + ("filename", "Closed Tickets - ABC.csv"), + ("csv-delimiter", ","), + ("undefined", "Export"), + ("__CSRFToken__", csrf_token), + ], + ) + eid = export_response.json()["eid"] + + logging.info(f"Waiting for OSTicket Export to complete (queue {queue_number})....") + csv_contents = download_export(session, eid) + attempts = 1 + while csv_contents is None: + csv_contents = download_export(session, eid) + attempts += 1 + time.sleep(5) + + if attempts > 20: + raise TimeoutError("Too many attempts to download export data") + + f = StringIO(csv_contents) + reader = csv.DictReader(f) + return list(reader) + + +def parse_node_text_to_install_number(node_text): + if not node_text: + return None + + try: + return int(node_text) + except ValueError: + modified_text = node_text.strip().replace("#", "").split(" ")[0] + try: + return int(modified_text) + except ValueError: + logging.error(f"Bad node: {node_text}") + return None + + +def import_ticket_numbers_from_osticket(): + attempts = 0 + while True: + try: + attempts += 1 + closed_tickets = attempt_to_load_osticket_data("8") + break + except Exception as e: + if attempts > 3: + raise e + + while True: + try: + attempts += 1 + open_tickets = attempt_to_load_osticket_data("1") + break + except Exception as e: + if attempts > 3: + raise e + + logging.info("Loading ticket numbers into install objects...") + for ticket in open_tickets + closed_tickets: + if ticket["node"]: + install_number = parse_node_text_to_install_number(ticket["node"]) + if install_number: + ticket_number = ticket["Ticket Number"] + install: Install = Install.objects.filter(install_number=install_number).first() + if install: + install.ticket_number = ticket_number + install.save() + + +if __name__ == "__main__": + import_ticket_numbers_from_osticket() diff --git a/src/meshdb/utils/spreadsheet_import/main.py b/src/meshdb/utils/spreadsheet_import/main.py index fd217f37..d8d15f98 100644 --- a/src/meshdb/utils/spreadsheet_import/main.py +++ b/src/meshdb/utils/spreadsheet_import/main.py @@ -8,6 +8,7 @@ import django from meshdb.utils.spreadsheet_import import logger +from meshdb.utils.spreadsheet_import.fetch_osticket_data import import_ticket_numbers_from_osticket logger.configure() @@ -228,6 +229,9 @@ def main(): logging.info(f"Importing links from UISP & '{links_path}'") load_links_supplement_with_uisp(get_spreadsheet_links((links_path))) + + logging.info(f"Importing ticket numbers from OSTicket") + import_ticket_numbers_from_osticket() except BaseException as e: if isinstance(e, KeyboardInterrupt): logging.error("Received keyboard interrupt, exiting early...") diff --git a/src/meshdb/utils/spreadsheet_import/parse_install.py b/src/meshdb/utils/spreadsheet_import/parse_install.py index b3873caf..a242e496 100644 --- a/src/meshdb/utils/spreadsheet_import/parse_install.py +++ b/src/meshdb/utils/spreadsheet_import/parse_install.py @@ -44,9 +44,7 @@ def create_install(row: SpreadsheetRow) -> Optional[models.Install]: install = models.Install( install_number=row.id, status=translate_spreadsheet_status_to_db_status(row.status), - ticket_id=None, - # TODO: Figure out if we can export data from OSTicket to back-fill this - # https://github.com/nycmeshnet/meshdb/issues/510 + ticket_number=None, request_date=row.request_date.date(), install_date=row.installDate, abandon_date=row.abandonDate,