From 9c94c1273432a640a7270c3206d74f5f2b4ce4e2 Mon Sep 17 00:00:00 2001 From: tim738745 <98717409+tim738745@users.noreply.github.com> Date: Thu, 11 Jul 2024 11:58:39 -0700 Subject: [PATCH] fix: 366 - (vin decoding jobs)/resolver fixes/changes (#368) --- django/api/services/decoded_vin_record.py | 12 +++++++----- django/api/services/resolvers.py | 3 ++- django/api/utilities/generic.py | 9 +++++++++ django/workers/external_apis/vinpower.py | 2 +- django/workers/external_apis/vpic.py | 2 +- django/workers/tasks.py | 14 +++++++++----- 6 files changed, 29 insertions(+), 13 deletions(-) diff --git a/django/api/services/decoded_vin_record.py b/django/api/services/decoded_vin_record.py index 9fd2304a..5ec6108e 100644 --- a/django/api/services/decoded_vin_record.py +++ b/django/api/services/decoded_vin_record.py @@ -13,7 +13,7 @@ def save_decoded_data( uploaded_vin_records, vins_to_insert, - decoded_records_to_update_map, + vins_to_decoded_record_ids_map, service_name, decoded_data, ): @@ -34,10 +34,12 @@ def save_decoded_data( decoded_records_to_insert.append( decoded_vin_model(vin=vin, data=decoded_datum) ) - elif vin in decoded_records_to_update_map: - decoded_record_to_update = decoded_records_to_update_map.get(vin) - decoded_record_to_update.update_timestamp = timezone.now() - decoded_record_to_update.data = decoded_datum + elif vin in vins_to_decoded_record_ids_map: + decoded_record_to_update = decoded_vin_model( + id=vins_to_decoded_record_ids_map[vin], + update_timestamp=timezone.now(), + data=decoded_datum, + ) decoded_records_to_update.append(decoded_record_to_update) elif vin in failed_vins: set_decode_successful(service_name, uploaded_record, False) diff --git a/django/api/services/resolvers.py b/django/api/services/resolvers.py index 510a8ee0..65b6785a 100644 --- a/django/api/services/resolvers.py +++ b/django/api/services/resolvers.py @@ -1,7 +1,8 @@ from dns.resolver import Resolver +from email_validator import caching_resolver def get_google_resolver(): resolver = Resolver() resolver.nameservers = ["8.8.8.8"] - return resolver + return caching_resolver(dns_resolver=resolver) diff --git a/django/api/utilities/generic.py b/django/api/utilities/generic.py index 20d5a8d7..98307551 100644 --- a/django/api/utilities/generic.py +++ b/django/api/utilities/generic.py @@ -4,3 +4,12 @@ def get_map(key_name, objects): key = getattr(object, key_name) result[key] = object return result + + +def get_unified_map(key_name, value_name, maps): + result = {} + for map in maps: + key = map.get(key_name) + value = map.get(value_name) + result[key] = value + return result diff --git a/django/workers/external_apis/vinpower.py b/django/workers/external_apis/vinpower.py index 98cb69e7..6c1adf86 100644 --- a/django/workers/external_apis/vinpower.py +++ b/django/workers/external_apis/vinpower.py @@ -14,7 +14,7 @@ def batch_decode(uploaded_vin_records): vins.append(record.vin) headers = {"content-type": "application/json"} response = requests.get(url, data=json.dumps(vins), headers=headers) - response.raise_for_status + response.raise_for_status() data = response.json() for vin in vins: diff --git a/django/workers/external_apis/vpic.py b/django/workers/external_apis/vpic.py index fa941a56..706df439 100644 --- a/django/workers/external_apis/vpic.py +++ b/django/workers/external_apis/vpic.py @@ -17,7 +17,7 @@ def batch_decode(uploaded_vin_records): body = {"format": "json", "data": request_data} response = requests.post(url, data=body) - response.raise_for_status + response.raise_for_status() data = response.json()["Results"] decoded_vins_map = {} for record in data: diff --git a/django/workers/tasks.py b/django/workers/tasks.py index 8fd67cbe..a14b451e 100644 --- a/django/workers/tasks.py +++ b/django/workers/tasks.py @@ -3,7 +3,7 @@ from api.models.uploaded_vins_file import UploadedVinsFile from api.models.uploaded_vin_record import UploadedVinRecord from api.constants.decoder import get_service -from api.utilities.generic import get_map +from api.utilities.generic import get_unified_map from api.services.decoded_vin_record import save_decoded_data from api.services.uploaded_vin_record import parse_and_save from django.db import transaction @@ -61,10 +61,14 @@ def batch_decode_vins(service_name, batch_size=50): for uploaded_record in uploaded_vin_records: uploaded_vins.add(uploaded_record.vin) vins_to_update = set() - decoded_records_to_update_map = get_map( - "vin", decoded_vin_model.objects.filter(vin__in=uploaded_vins) + vins_to_decoded_record_ids_map = get_unified_map( + "vin", + "id", + decoded_vin_model.objects.only("id", "vin") + .filter(vin__in=uploaded_vins) + .values(), ) - for decoded_vin in decoded_records_to_update_map: + for decoded_vin in vins_to_decoded_record_ids_map: vins_to_update.add(decoded_vin) vins_to_insert = uploaded_vins.difference(vins_to_update) @@ -74,7 +78,7 @@ def batch_decode_vins(service_name, batch_size=50): save_decoded_data( uploaded_vin_records, vins_to_insert, - decoded_records_to_update_map, + vins_to_decoded_record_ids_map, service_name, decoded_data, )