From f01dd50a23121b12a4fefd69684b43124bfbfd8b Mon Sep 17 00:00:00 2001 From: pphung <45565029+p-phung@users.noreply.github.com> Date: Fri, 22 Nov 2024 11:48:36 +0100 Subject: [PATCH 1/2] rename workflow file for dev --- .github/workflows/update-image-dev.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/update-image-dev.yaml b/.github/workflows/update-image-dev.yaml index 4eba0ad..16919b6 100644 --- a/.github/workflows/update-image-dev.yaml +++ b/.github/workflows/update-image-dev.yaml @@ -1,4 +1,4 @@ -name: Update pipeline prod +name: Update pipeline dev on: push: From 7fd100cfcbb1c98fbde4a9c04d94d23bd17ad8db Mon Sep 17 00:00:00 2001 From: pphung <45565029+p-phung@users.noreply.github.com> Date: Fri, 22 Nov 2024 13:08:52 +0100 Subject: [PATCH 2/2] add readme, change default dateend --- README.md | 31 ++++++++++++++++++++++++++++++- nrt_rainfall_pipeline.py | 2 +- nrt_rainfall_pipeline/pipeline.py | 2 +- 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 57c94f3..3525099 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,32 @@ # Near real-time rainfall pipeline -TBD \ No newline at end of file +Near real-time rainfall monitoring. Part of the project RIPOSTE for Cameroon Red Cross Society. + +## Description +The pipeline roughly consists of three steps: + +- Extract daily data on rainfall measurement of the past days (e.g. last 14 days) from [NOAA's GPM](https://gpm.nasa.gov/data/directory) +- Transform the data into pre-defined areas (health districts) and calculate average rainfall of the past days. Then determine which area has its average rainfall higher than pre-defined thresholds. +- Send this data as alert to the EspoCRM for the NS. + +## Basic Usage +To run the pipeline locally + +1. Fill in the secrets in .env.example and rename the file to .env; in this way, they will be loaded as environment variables +2. Install requirements + ``` + pip install poetry + poetry install --no-interaction + ``` +3. Run the pipeline : `python nrt_rainfall_pipeline.py --extract --transform --send` + ``` + Usage: nrt_rainfall_pipeline.py [OPTIONS] + + Options: + --country TEXT country ISO3 + --extract extract rainfall data + --transform calculate rainfall data into pre-defined + --send send to IBF app + --dateend specify date until which the data should be extracted + --help Show this message and exit. + ``` diff --git a/nrt_rainfall_pipeline.py b/nrt_rainfall_pipeline.py index 891bc52..7cdc48d 100644 --- a/nrt_rainfall_pipeline.py +++ b/nrt_rainfall_pipeline.py @@ -14,7 +14,7 @@ @click.option( "--dateend", help="date start in YYYY-mm-dd", - default=(datetime.now(timezone.utc)-timedelta(days=1)).strftime("%Y-%m-%d"), + default=(datetime.now(timezone.utc)-timedelta(days=2)).strftime("%Y-%m-%d"), ) def run_nrt_rainfall_pipeline( diff --git a/nrt_rainfall_pipeline/pipeline.py b/nrt_rainfall_pipeline/pipeline.py index 08855f0..d4b5e06 100644 --- a/nrt_rainfall_pipeline/pipeline.py +++ b/nrt_rainfall_pipeline/pipeline.py @@ -39,7 +39,7 @@ def run_pipeline( transform: bool = True, send: bool = True, save: bool =True, - dateend: datetime = datetime.now(timezone.utc)-timedelta(days=1), + dateend: datetime = datetime.now(timezone.utc)-timedelta(days=2), # debug: bool = False ): """Run the flood data pipeline"""