completed unittests

.gitignore

@@ -1 +1 @@
-/.DS_Store
+**/.DS_Store

SVInsight/svi.py

@@ -5,6 +5,7 @@
 import shutil
 import zipfile
 import numpy as np
+import requests
 import pandas as pd
 from ftplib import FTP
 import ftputil
@@ -173,7 +174,8 @@ def boundaries_data(self, boundary: str = 'bg', year: int = 2019, overwrite: boo
 
         # If shapefile already exists, pass 
         if os.path.exists(os.path.join(self.boundaries, f"{self.project_name}_{year}_{boundary}.gpkg")) and overwrite is False:
-            pass
+
+            return gpd.read_file(os.path.join(self.boundaries, f"{self.project_name}_{year}_{boundary}.gpkg"))
         else:
 
             # create final output geodataframe
@@ -212,12 +214,21 @@ def _download(year, state, boundary):
                 #     ftp.retrbinary(f"RETR {zipped_filename}", file.write)
                 # print('opened binary')
 
-                # Connect to the FTP server
-                with ftputil.FTPHost('ftp2.census.gov', 'anonymous') as ftp_host:
-                    # Change the current directory
-                    ftp_host.chdir(f'geo/tiger/GENZ{year}/shp/')
-                    # Open the remote file and write it to a local file
-                    ftp_host.download(zipped_filename, zipped_dir)
+                # Specify the URL of the file you want to download
+                url = f"https://www2.census.gov/geo/tiger/GENZ{year}/shp/{zipped_filename}"
+
+                # Send a GET request to the URL
+                response = requests.get(url, stream=True)
+
+                # Check if the request was successful
+                if response.status_code == 200:
+                    # Open a local file with the same name as the remote file
+                    with open(zipped_dir, 'wb') as file:
+                        # Write the contents of the remote file to the local file
+                        for chunk in response.iter_content(chunk_size=1024):
+                            if chunk:
+                                file.write(chunk)
+
 
                 # Unzip the file
                 shutil.unpack_archive(zipped_dir, filename_dir)
@@ -725,7 +736,7 @@ def _calc_variance(fa, n_factors, loading_factors, facs, df_variances):
 
         # save geopackage and csv output
         output_df.to_file(os.path.join(self.svis, f"{self.project_name}_{year}_{boundary}_{config_file}_svi.gpkg"))
-        data_df.to_csv(os.path.join(self.svis, f"{self.project_name}_{year}_{boundary}_{config_file}_svi.csv"))
+        output_df.to_csv(os.path.join(self.svis, f"{self.project_name}_{year}_{boundary}_{config_file}_svi.csv"))
 
 
 
@@ -865,7 +876,7 @@ def _fill_empty(self, data_df, boundary, geoids, api_key, year, verbose):
             cols_to_drop = data_df.filter(like='_y', axis=1).columns
             data_df = data_df.drop(cols_to_drop, axis=1)
 
-        if verbose is True:
+        if verbose is True and empty_cols:
              with open(empty_output_name, 'w', newline='') as file:
                 writer = csv.writer(file)
                 # Write rows to the CSV file

examples/svinsight_example_Travis_County.py

@@ -25,26 +25,26 @@
 # extract shapefile
 export = test.boundaries_data(boundary, year, overwrite=True)
 
-# extract raw census data
-test.census_data(boundary, 
-                    year, 
-                    interpolate=True,
-                    verbose=True,
-                    overwrite=True)
-
-# # add variable
-# test2.add_variable(boundary,
-#                    year,
-#                    'newvar',
-#                    ['B03002_021E'])
-
-
-# # print(test2.all_vars_eqs['MEDAGE']['description'])
-# # test2.var_descriptions(['MEDAGE','PPUNIT'])
-
-# # configure run and calculate svi
-test.configure_variables(config_file)
-test.calculate_svi(config_file, boundary, year)
+# # extract raw census data
+# test.census_data(boundary, 
+#                     year, 
+#                     interpolate=True,
+#                     verbose=True,
+#                     overwrite=True)
+
+# # # add variable
+# # test2.add_variable(boundary,
+# #                    year,
+# #                    'newvar',
+# #                    ['B03002_021E'])
+
+
+# # # print(test2.all_vars_eqs['MEDAGE']['description'])
+# # # test2.var_descriptions(['MEDAGE','PPUNIT'])
+
+# # # configure run and calculate svi
+# test.configure_variables(config_file)
+# test.calculate_svi(config_file, boundary, year)
 
 
 

tests/test_project_multiple_counties/Data/test_project_multiple_counties_2015_bg_missing_columns.csv

@@ -0,0 +1 @@
+B27001,bg empty

tests/test_project_multiple_counties/Data/test_project_multiple_counties_2015_bg_missing_values.csv

@@ -0,0 +1,112 @@
+211170636032,B25064_001E,Interpolated
+211170636051,B25064_001E,Interpolated
+211170636061,B25064_001E,Interpolated
+211170640003,B25077_001E,Interpolated
+211170641002,B25064_001E,Interpolated
+211170641003,B25064_001E,Interpolated
+211170653001,B25064_001E,Interpolated
+211170653003,B25064_001E,Interpolated
+211170653005,B25064_001E,Interpolated
+211170655021,B25064_001E,Interpolated
+211170655022,B25064_001E,Interpolated
+211170670002,B25077_001E,Interpolated
+484530001022,B25064_001E,Interpolated
+484530002031,B25077_001E,Interpolated
+484530005001,B25077_001E,Interpolated
+484530006011,B25077_001E,Interpolated
+484530006012,B25077_001E,Interpolated
+484530006032,B25077_001E,Interpolated
+484530006033,B25077_001E,Interpolated
+484530006034,B25077_001E,Interpolated
+484530006035,B25077_001E,Interpolated
+484530006041,B25077_001E,Interpolated
+484530006042,B25077_001E,Interpolated
+484530006043,B25077_001E,Interpolated
+484530015012,B25064_001E,Interpolated
+484530015013,B25064_001E,Interpolated
+484530015014,B25064_001E,Interpolated
+484530016023,B25077_001E,Interpolated
+484530016033,B25064_001E,Interpolated
+484530016042,B25064_001E,Interpolated
+484530017052,B25064_001E,Interpolated
+484530017142,B25064_001E,Interpolated
+484530017143,B25064_001E,Interpolated
+484530017192,B25064_001E,Interpolated
+484530017222,B25077_001E,Interpolated
+484530017332,B25064_001E,Interpolated
+484530017382,B25064_001E,Interpolated
+484530017383,B25064_001E,Interpolated
+484530017451,B25064_001E,Interpolated
+484530017492,B25064_001E,Interpolated
+484530017493,B25077_001E,Interpolated
+484530017501,B25077_001E,Interpolated
+484530017521,B25077_001E,Interpolated
+484530017522,B25077_001E,Interpolated
+484530017531,B25077_001E,Interpolated
+484530017571,B25064_001E,Interpolated
+484530017702,B25064_001E,Interpolated
+484530017741,B25064_001E,Interpolated
+484530017743,B25064_001E,Interpolated
+484530017752,B25064_001E,Interpolated
+484530017791,B25064_001E,Interpolated
+484530017792,B25064_001E,Interpolated
+484530017811,B25064_001E,Interpolated
+484530017852,B25077_001E,Interpolated
+484530017861,B25077_001E,Interpolated
+484530017863,B25077_001E,Interpolated
+484530018043,B25077_001E,Interpolated
+484530018064,B25077_001E,Interpolated
+484530018121,B25077_001E,Interpolated
+484530018122,B25077_001E,Interpolated
+484530018192,B25077_001E,Interpolated
+484530018224,B25077_001E,Interpolated
+484530018413,B25064_001E,Interpolated
+484530018491,B25077_001E,Interpolated
+484530018492,B25077_001E,Interpolated
+484530018493,B25077_001E,Interpolated
+484530018494,B25077_001E,Interpolated
+484530018503,B25077_001E,Interpolated
+484530018531,B25064_001E,Interpolated
+484530018541,B25064_001E,Interpolated
+484530018551,B25064_001E,Interpolated
+484530018555,B25064_001E,Interpolated
+484530018581,B25064_001E,Interpolated
+484530018582,B25064_001E,Interpolated
+484530018583,B25064_001E,Interpolated
+484530018584,B25064_001E,Interpolated
+484530018585,B25064_001E,Interpolated
+484530018591,B25064_001E,Interpolated
+484530018631,B25077_001E,Interpolated
+484530018632,B25077_001E,Interpolated
+484530018642,B25077_001E,Interpolated
+484530019142,B25064_001E,Interpolated
+484530019162,B25064_001E,Interpolated
+484530019171,B25064_001E,Interpolated
+484530019172,B25064_001E,Interpolated
+484530019173,B25064_001E,Interpolated
+484530019174,B25064_001E,Interpolated
+484530019181,B25064_001E,Interpolated
+484530020052,B25077_001E,Interpolated
+484530021063,B25064_001E,Interpolated
+484530023041,B25077_001E,Interpolated
+484530023043,B25077_001E,Interpolated
+484530023083,B25077_001E,Interpolated
+484530023084,B25077_001E,Interpolated
+484530023141,B25077_001E,Interpolated
+484530023142,B25077_001E,Interpolated
+484530023144,B25077_001E,Interpolated
+484530023152,B25077_001E,Interpolated
+484530023161,B25077_001E,Interpolated
+484530023162,B25077_001E,Interpolated
+484530023163,B25077_001E,Interpolated
+484530023172,B25077_001E,Interpolated
+484530024024,B25077_001E,Interpolated
+484530024072,B25064_001E,Interpolated
+484530024191,B25077_001E,Interpolated
+484530024192,B25077_001E,Interpolated
+484530024212,B25064_001E,Interpolated
+484530024301,B25064_001E,Interpolated
+484530024302,B25064_001E,Interpolated
+484530024351,B25064_001E,Interpolated
+484530025001,B25064_001E,Interpolated
+484530023143,B25077_001E,Tract Filled