feat: adds error handling for remainder of scripts (#153)

django/api/services/arc_project_tracking.py

@@ -51,8 +51,8 @@ def import_from_xls(excel_file):
 
     try:
         for _, row in df.iterrows():
-            if row["Publicly Announced"] == '': continue # Skip rows without this field
             row_count += 1
+            if row["Publicly Announced"] == '': continue # Skip rows without this field
             ARCProjectTracking.objects.create(
                 funding_call=row["Funding Call"],
                 proponent=row["Proponent"],

django/api/services/charger_rebates.py

@@ -28,8 +28,6 @@ def import_from_xls(excel_file):
     ]), axis=1, inplace=True)
     df = trim_all_columns(df)
     df = df.applymap(lambda s: s.upper() if type(s) == str else s)
-
-    # df.fillna('')
     df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna(''))
     try:
         for _, row in df.iterrows():

django/api/services/data_fleets.py

@@ -11,6 +11,7 @@ def trim_all_columns(df):
 
 
 def import_from_xls(excel_file):
+    row_count = 1
     df = pd.read_excel(excel_file, 'Data Fleets')
     df.drop(df.columns.difference([
         "Current Stage",
@@ -47,6 +48,7 @@ def import_from_xls(excel_file):
     df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna(''))
 
     for _, row in df.iterrows():
+        row_count += 1
         try:
             DataFleets.objects.create(
                 current_stage=row["Current Stage"],
@@ -77,6 +79,5 @@ def import_from_xls(excel_file):
                 potential_rebate=row["Potential Rebate"]
             )
         except Exception as error:
-            print(error)
-            print(row)
+            return (error,'data',row_count) 
     return True

django/api/services/hydrogen_fleets.py

@@ -11,6 +11,7 @@ def trim_all_columns(df):
 
 
 def import_from_xls(excel_file):
+    row_count = 1
     df = pd.read_excel(excel_file, 'Fleets')
     df.drop(df.columns.difference([
         "Application #",
@@ -32,8 +33,9 @@ def import_from_xls(excel_file):
     df = trim_all_columns(df)
     df = df.applymap(lambda s: s.upper() if type(s) == str else s)
     df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna(''))
-    for _, row in df.iterrows():
-        try:
+    try:
+        for _, row in df.iterrows():
+            row_count += 1
             HydrogenFleets.objects.create(
                 application_number=row["Application #"],
                 fleet_number=row["Fleet #"],
@@ -51,7 +53,6 @@ def import_from_xls(excel_file):
                 dealer_name=row["Dealer Name"],
                 rebate_amount=row["Rebate Amount"]
             )
-        except Exception as error:
-            print("error: ", error)
-            print(row)
+    except Exception as error:
+        return (error,'data',row_count)
     return True

django/api/services/ldv_rebates.py

@@ -11,6 +11,7 @@ def trim_all_columns(df):
 
 
 def import_from_xls(excel_file):
+    row_count = 1
     df = pd.read_excel(excel_file, 'Raw Data')
     df = trim_all_columns(df)
     df = df.applymap(lambda s: s.upper() if type(s) == str else s)
@@ -47,8 +48,8 @@ def import_from_xls(excel_file):
     )
     df.fillna('')
 
-    for _, row in df.iterrows():
-        try:
+    try:
+        for _, row in df.iterrows():
             LdvRebates.objects.create(
                 casl_consent=row["CASL Consent"],
                 date_approved=row["DATE APPROVED"],
@@ -78,7 +79,6 @@ def import_from_xls(excel_file):
                 delivered=row["Delivered"],
                 consent_to_contact=row["Consent to Contact"]
             )
-        except Exception as error:
-            print(error)
-            print(row)
+    except Exception as error:
+        return (error,'data',row_count)
     return True

django/api/services/public_charging.py

@@ -11,6 +11,7 @@ def trim_all_columns(df):
 
 
 def import_from_xls(excel_file):
+    row_count = 3
     df = pd.read_excel(excel_file, 'Project_applications', header=2)
     df = trim_all_columns(df)
     df = df.applymap(lambda s: s.upper() if type(s) == str else s)
@@ -25,12 +26,11 @@ def import_from_xls(excel_file):
         value=True,
         inplace=True
     )
-    for _, row in df.iterrows():
-        try:
+    try:
+        for _, row in df.iterrows():
+            row_count += 1
             PublicCharging.objects.create(
                 applicant_name=row["Applicant Name"],
-                # city=row["City"],
-                # postal_code=row["Postal Code"],
                 address=row["Address"],
                 charging_station_info=row["Charging Station Info"],
                 between_25kw_and_50kw=row[">25kW; <50kW"],
@@ -48,9 +48,6 @@ def import_from_xls(excel_file):
                 review_number=row["Review Number"],
                 rebate_paid=row["Paid out rebate amount"],
             )
-        except Exception as error:
-            print('-----------------------')
-            print(error)
-            print(row)
-            print('-----------------------')
+    except Exception as error:
+        return (error,'data',row_count) 
     return True

django/api/services/scrap_it.py

@@ -11,16 +11,18 @@ def trim_all_columns(df):
 
 
 def import_from_xls(excel_file):
+    row_count = 6
     df = pd.read_excel(excel_file, 'TOP OTHER TRANSACTIONS', header=5)
 
     df = trim_all_columns(df)
     df = df.applymap(lambda s: s.upper() if type(s) == str else s)
 
     df = df.apply(lambda x: x.fillna(0) if x.dtype.kind in 'biufc' else x.fillna(''))
 
-    for _, row in df.iterrows():
-        if row["VIN"] == '': continue # Skip rows without this field
-        try:
+    try:
+        for _, row in df.iterrows():
+            row_count += 1
+            if row["VIN"] == '': continue # Skip rows without this field
             ScrapIt.objects.create(
                 approval_number=row["Approval Num"],
                 application_received_date=row["App Recv'd Date"],
@@ -34,7 +36,6 @@ def import_from_xls(excel_file):
                 budget_code=row["Budget Code"],
                 scrap_date=row["Scrap Date"]
             )
-        except Exception as error:
-            print(error)
-            print(row)
+    except Exception as error:
+        return (error,'data',row_count)
     return True

django/api/services/speciality_use_vehicle_incentives.py

@@ -22,6 +22,7 @@ def applicant_type(row):
 
 
 def import_from_xls(excel_file):
+    row_count = 1
     df = pd.read_excel(excel_file, 'Sheet1')
     df.drop(df.columns.difference([
         "Approvals",
@@ -46,8 +47,9 @@ def import_from_xls(excel_file):
     df = df.applymap(lambda s: s.upper() if type(s) == str else s)
     df['Applicant Type'] = df.apply(lambda row: applicant_type(row), axis=1)
 
-    for _, row in df.iterrows():
-        try:
+    try:
+        for _, row in df.iterrows():
+            row_count += 1
             SpecialityUseVehicleIncentives.objects.create(
                 approvals=row["Approvals"],
                 date=row["Date"],
@@ -60,7 +62,6 @@ def import_from_xls(excel_file):
                 manufacturer=row["Manufacturer"],
                 model=row["Model"],
             )
-        except Exception as error:
-            print(error)
-            print(row)
+    except Exception as error:
+        return (error,'data',row_count)  
     return True

django/api/viewsets/upload.py

@@ -107,6 +107,7 @@ def import_data(self, request):
             done = (error, 'file')
         final_count = model.objects.all().count()
         records_inserted = final_count - starting_count
+        records_inserted_msg = "{} records inserted. This table currently contains {} records.".format(records_inserted, final_count)
         if done != True:
             try:
                 error_location = done[1]
@@ -120,7 +121,7 @@ def import_data(self, request):
                 if error_location == 'data':
                     if error_type in (type(LookupError), type(KeyError), 'KeyError') :
                         error_msg = "Please make sure you've uploaded a file with the correct data including the correctly named columns. There was an error finding: {}. This dataset requires the following columns: {}".format(error, field_names)
-                    elif error_type == type(ValueError):
+                    elif error_type == 'ValueError' or type(ValueError):
                         ## note for next batch of scripts, possibly add str(type(ValueError)) 
                         ## to this but check for impacts to other exceptions
                         error_msg = "{} on row {}. Please make sure you've uploaded a file with the correct data.".format(error, error_row)
@@ -130,10 +131,10 @@ def import_data(self, request):
                     error_msg = "{}. Please make sure you've uploaded a file with the correct data including the correctly named worksheets.".format(error)
                 if error_msg[-1] != '.':
                     error_msg+='.'
-                error_msg += " {} records inserted.".format(records_inserted)
+                error_msg += records_inserted_msg
                 return Response(error_msg, status=status.HTTP_400_BAD_REQUEST)
             except Exception as error:
                 print(error)
                 return Response('There was an issue!', status=status.HTTP_400_BAD_REQUEST)
         else:
-            return Response("{} records inserted.".format(records_inserted), status=status.HTTP_201_CREATED)
+            return Response(records_inserted_msg, status=status.HTTP_201_CREATED)