From ddc17db315e401eb72aa9e09942a468c3423dd6b Mon Sep 17 00:00:00 2001 From: pphung <45565029+p-phung@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:21:56 +0100 Subject: [PATCH 1/3] correct readme, correct click.option's help msg, change days-to-observe to 4 --- README.md | 49 +++++++++++++++++++++++----------------- config/config.yaml | 2 +- nrt_rainfall_pipeline.py | 8 +++---- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 2105dc4..d3c5566 100644 --- a/README.md +++ b/README.md @@ -23,21 +23,34 @@ To run the pipeline locally pip install poetry poetry install --no-interaction ``` -3. Run the pipeline : `python nrt_rainfall_pipeline.py --extract --transform --send` +3. Edit the configuration in `config\config.yaml` + ``` + - name: + days-to-observe: # number of most recent days to observe rainfall e.g. 14 + alert-on-threshold: # threshold to send to EspoCRM e.g. 50 + shapefile-area: .geojson # shapefile of areas (. geojson) where the zonal stats bases on + espo-area: # entity storing areas code (and id) + entity: + field: + espo-destination: # entity to send alerts to + entity: + field: + ``` +4. Run the pipeline : `python nrt_rainfall_pipeline.py --extract --transform --send` ``` Usage: nrt_rainfall_pipeline.py [OPTIONS] Options: --country TEXT country ISO3 - --extract extract rainfall data - --transform calculate rainfall data into pre-defined - --send send to IBF app - --dateend specify date until which the data should be extracted - --help Show this message and exit. + --extract extract NRT rainfall raster data + --transform calculate rainfall data in pre-defined administrative areas + --send send to EspoCRM + --dateend specify a customed latest date YYYY-mm-dd until which the data should be extracted, by default it is the date before today + --help Show this message and exit ``` -Payload sent to EspoCRM: - +__Note:__ Payload sent to EspoCRM +``` { "status": "onhold", "type": "heavyrainfall", @@ -45,6 +58,12 @@ Payload sent to EspoCRM: "": "", "": "" } +``` +Where: +- ``: is the value of `espo-area`'s `field` in the config +- ``: is to be automatically filled in the pipeline +- ``: is the value of `espo-destination`'s `field` in the config +- ``: is to be automatically filled in the pipeline ## Adding new country 1. Prepare shapefile @@ -65,17 +84,5 @@ Payload sent to EspoCRM: | `type` | `enum` | `heavyrainfall` | | `source` | `enum` or `text` | `GPM` | - Make sure the entity for area is linked with this one -4. Add the new country to the `config\config.yaml` below the existing one: - ``` - - name: - days-to-observe: 14 # number of most recent days to observe rainfall - alert-on-threshold: 50 # threshold to send to EspoCRM - shapefile-area: .geojson # shapefile of areas (. geojson) where the zonal stats bases on - espo-area: # entity storing areas code (and id) - entity: - field: - espo-destination: # entity to send alerts to - entity: - field: - ``` +4. Add the new country to the `config\config.yaml`, see Step 3 in __Basic usage__: 5. Test and adjust settings if needed. \ No newline at end of file diff --git a/config/config.yaml b/config/config.yaml index d89381d..950a9ed 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -1,6 +1,6 @@ countries: - name: CMR - days-to-observe: 14 # number of most recent days to observe rainfall + days-to-observe: 4 # number of most recent days to observe rainfall alert-on-threshold: 50 # threshold to send to EspoCRM shapefile-area: cmr_district_sante_2022.geojson # shapefile of areas (.geojson) where the zonal stats bases on espo-area: # entity storing areas code and id diff --git a/nrt_rainfall_pipeline.py b/nrt_rainfall_pipeline.py index 7cdc48d..3a92287 100644 --- a/nrt_rainfall_pipeline.py +++ b/nrt_rainfall_pipeline.py @@ -8,17 +8,17 @@ @click.command() @click.option("--country", help="country ISO3", default="CMR") @click.option("--extract", help="extract NRT rainfall raster data", default=False, is_flag=True) -@click.option("--transform", help="calculate rainfall per admin", default=False, is_flag=True) -@click.option("--send", help="send to EspoCRM", default=False, is_flag=True) +@click.option("--transform", help="calculate rainfall data in pre-defined administrative areas", default=False, is_flag=True) +@click.option("--send", help="specify a customed latest date YYYY-mm-dd until which the data should be extracted, by default it is the date before today", default=False, is_flag=True) @click.option("--save", help="save to storage", default=False, is_flag=True) @click.option( "--dateend", - help="date start in YYYY-mm-dd", + help="date end in YYYY-mm-dd", default=(datetime.now(timezone.utc)-timedelta(days=2)).strftime("%Y-%m-%d"), ) def run_nrt_rainfall_pipeline( - country, extract, transform, send, save, dateend#, debug + country, extract, transform, send, save, dateend ): dateend = datetime.strptime(dateend, "%Y-%m-%d") pipe = Pipeline( From 467ea3576b399da98e8092ac0282741488b15405 Mon Sep 17 00:00:00 2001 From: pphung <45565029+p-phung@users.noreply.github.com> Date: Thu, 19 Dec 2024 11:37:46 +0100 Subject: [PATCH 2/3] remove print func --- nrt_rainfall_pipeline/transform.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nrt_rainfall_pipeline/transform.py b/nrt_rainfall_pipeline/transform.py index 291293d..723f899 100644 --- a/nrt_rainfall_pipeline/transform.py +++ b/nrt_rainfall_pipeline/transform.py @@ -111,7 +111,6 @@ def __prepare_data_for_espo(self, stats): new_d[destination_field] = new_d.pop('median') new_d.update(additional_data) stats_list.append(new_d) - print('stats_list: ', stats_list) threshold = self.settings.get_country_setting(self.country, "alert-on-threshold") filtered = self.__filter_dict(stats_list, destination_field, threshold) From 53133b170ac8f2953596f95e747fd1a4c5618bcf Mon Sep 17 00:00:00 2001 From: pphung <45565029+p-phung@users.noreply.github.com> Date: Thu, 19 Dec 2024 15:54:45 +0100 Subject: [PATCH 3/3] correct entity field for admin areas --- config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.yaml b/config/config.yaml index 950a9ed..8408327 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -5,7 +5,7 @@ countries: shapefile-area: cmr_district_sante_2022.geojson # shapefile of areas (.geojson) where the zonal stats bases on espo-area: # entity storing areas code and id entity: CHealthDistrict - field: CHealthDistrictId + field: cHealthDistrictId espo-destination: # entity to send alerts to entity: CClimaticHazard field: averageRainfall \ No newline at end of file