Ridership gh action

.github/workflows/ridership-action.yml

@@ -0,0 +1,33 @@
+name: Automate ridership data updates
+
+on:
+  workflow_dispatch:
+
+  schedule:
+    # Run monthly at 12:30pm CST which is 5:30pm UTC
+    - cron: 30 17 1 * * 
+
+env: 
+  PYTHON_VERSION: 3.10.6
+  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+
+
+jobs:
+  download-ridership-data:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+
+      - name: Download and save ridership data to s3
+
+        run: |
+          pip install -r requirements.txt
+          python -c 'from scrape_data.ridership_download import save_ridership_json; \
+            save_ridership_json()' \
+            $AWS_ACCESS_KEY_ID $AWS_SECRET_ACCESS_KEY

data_analysis/ridership_to_json.py

@@ -64,7 +64,8 @@ def get_latest_month_and_year(ridership_df: pd.DataFrame) -> tuple:
     return latest_date.month, latest_date.year
 
 
-def ridership_to_json(ridership_df: pd.DataFrame, month: int = None, year: int = None) -> None:
+def ridership_to_json(ridership_df: pd.DataFrame, month: int = None, year: int = None,
+                      save: bool = True) -> None:
     """
     Save ridership data to JSON for given month and year.
     Note that the data is typically a few months 
@@ -83,6 +84,7 @@ def ridership_to_json(ridership_df: pd.DataFrame, month: int = None, year: int =
         4	9	01/01/2001	U	11207
         month (int): Month of interest. Defaults to None
         year (int): Year of interest. Defaults to None
+        save (bool): Whether to save JSON locally. Defaults to True.
     """
     ridership = ridership_df.copy()
     latest_month, latest_year = get_latest_month_and_year(ridership)
@@ -119,21 +121,24 @@ def ridership_to_json(ridership_df: pd.DataFrame, month: int = None, year: int =
     df_daytype_summary_json = df_daytype_summary.to_json(orient='records')
     full_json = {'date': f'{month_name} {year}'}
     full_json['data'] = json.loads(df_daytype_summary_json)
-    with open(DATA_PATH / f'{month_name}_{year}_cta_ridership_data_day_type_summary.json', 'w') as outfile:
-        json.dump(full_json, outfile)
-
+    if save:
+        with open(DATA_PATH / f'{month_name}_{year}_cta_ridership_data_day_type_summary.json', 'w') as outfile:
+            json.dump(full_json, outfile)
+    else:
+        return json.dumps(full_json, indent=4)
+
 app = typer.Typer()
 
 @app.command()
-def main(month: int = None, year: int = None) -> None:
+def main(month: int = None, year: int = None, save: bool = True) -> None:
 
     print("Loading data from data.cityofchicago.org")
     ridership_df = pd.read_csv(
         'https://data.cityofchicago.org/api/views/'
         'jyb9-n7fm/rows.csv?accessType=DOWNLOAD'
     )
     print("Done!")
-    ridership_to_json(ridership_df=ridership_df, month=month, year=year)
+    return ridership_to_json(ridership_df=ridership_df, month=month, year=year, save=save)
 
 
 if __name__ == '__main__':

scrape_data/ridership_download.py

@@ -0,0 +1,46 @@
+import boto3
+import sys
+import data_analysis.ridership_to_json as ridership_to_json
+import data_analysis.static_gtfs_analysis as sga
+
+ACCESS_KEY = sys.argv[1]
+SECRET_KEY = sys.argv[2]
+
+client = boto3.client(
+    's3',
+    aws_access_key_id=ACCESS_KEY,
+    aws_secret_access_key=SECRET_KEY
+)
+
+s3 = boto3.resource(
+    's3',
+    region_name='us-east-1',
+    aws_access_key_id=ACCESS_KEY,
+    aws_secret_access_key=SECRET_KEY
+)
+
+def save_ridership_json() -> None:
+    ridership_json = ridership_to_json.main(save=False)
+    s3_ridership_json_path = 'frontend_data_files/cta_ridership_data_day_type_summary.json'
+    print(f'Saving {s3_ridership_json_path}')
+    s3.Object(
+        sga.BUCKET,
+        f'{s3_ridership_json_path}')\
+        .put(Body=ridership_json)
+
+    # Check that the file was uploaded successfully
+    keys(sga.BUCKET, [s3_ridership_json_path])
+
+
+# https://stackoverflow.com/questions/30249069/listing-contents-of-a-bucket-with-boto3
+def keys(bucket_name: str, filenames: list,
+         prefix: str='/', delimiter: str='/',
+         start_after: str='') -> None:
+    s3_paginator = client.get_paginator('list_objects_v2')
+    prefix = prefix.lstrip(delimiter)
+    start_after = (start_after or prefix) if prefix.endswith(delimiter) else start_after
+    for page in s3_paginator.paginate(Bucket=bucket_name, Prefix=prefix, StartAfter=start_after):
+        for content in page.get('Contents', ()):
+            if content['Key'] in filenames:
+                print(f"{content['Key']} exists")
+