Generate consolidated csv. IMPORTS=scripts/us_usda/quickstats:UsdaAgS…

…urvey (datacommonsorg#879)
dsfsi · Aug 8, 2023 · 3e80d42 · 3e80d42
1 parent aed0a86
commit 3e80d42
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 6 deletions.
diff --git a/scripts/us_usda/quickstats/manifest.json b/scripts/us_usda/quickstats/manifest.json
@@ -8,7 +8,7 @@
             "scripts": ["process.py"],
             "import_inputs": [{
                 "template_mcf": "ag.tmcf",
-                "cleaned_csv": "output/ag-2023.csv"
+                "cleaned_csv": "output/consolidated.csv"
             }],
         "cron_schedule": "0 10 * * 2"
         }

diff --git a/scripts/us_usda/quickstats/process.py b/scripts/us_usda/quickstats/process.py
@@ -103,10 +103,14 @@ def get_response_file_path(out_dir, year, county):
     return f"{get_response_dir(out_dir, year)}/{county}.json"
 
 
+def get_year_csv_file_path(out_dir, year):
+    return f"{out_dir}/ag-{year}.csv"
+
+
 def write_aggregate_csv(year, out_dir):
     parts_dir = get_parts_dir(out_dir, year)
     part_files = os.listdir(parts_dir)
-    out_file = f"{out_dir}/ag-{year}.csv"
+    out_file = get_year_csv_file_path(out_dir, year)
 
     print('Writing aggregate CSV', out_file)
 
@@ -121,6 +125,21 @@ def write_aggregate_csv(year, out_dir):
                     csv_writer.writerows(csv.DictReader(part))
 
 
+def write_consolidated_csv(years, out_dir):
+    out_file = f"{out_dir}/consolidated.csv"
+
+    print('Writing consolidated CSV', out_file)
+
+    with open(out_file, 'w', newline='') as out:
+        csv_writer = csv.DictWriter(out,
+                                    fieldnames=CSV_COLUMNS,
+                                    lineterminator='\n')
+        csv_writer.writeheader()
+        for year in years:
+            with open(get_year_csv_file_path(out_dir, year), 'r') as part:
+                csv_writer.writerows(csv.DictReader(part))
+
+
 def fetch_and_write(county_name, year, svs, out_dir):
     out_file = f"{get_parts_dir(out_dir, year)}/{county_name.replace('[^a-zA-Z0-9]', '')}.csv"
     api_data = get_survey_county_data(year, county_name, out_dir)
@@ -255,9 +274,13 @@ def get_multiple_years():
     start = datetime.now()
     print('Start', start)
 
+    out_dir = "output"
     svs = load_svs()
-    for year in range(2000, datetime.now().year + 1):
-        process_survey_data(year, svs, "output")
+    years = range(2000, datetime.now().year + 1)
+    for year in years:
+        process_survey_data(year, svs, out_dir)
+
+    write_consolidated_csv(years, out_dir)
 
     end = datetime.now()
     print('End', end)
@@ -284,8 +307,7 @@ def get_usda_api_key():
 def main(_):
     load_usda_api_key()
     print('USDA API key', get_usda_api_key())
-    # get_multiple_years()
-    get_all_counties()
+    get_multiple_years()
 
 
 if __name__ == '__main__':