IMPA creating pipelines using poetry

.flake8

@@ -1,2 +1,7 @@
 [flake8]
-max-line-length = 100
+max-line-length = 120
+
+exclude =
+ **/rionowcast/gypscie/*
+ **/rionowcast/test_flow/*
+ script.py

.github/workflows/cd_staging.yaml

@@ -40,7 +40,7 @@ jobs:
 
  - name: Install Python dependencies for deploying
  run: |-
- pip install -U pip poetry
+ pip install -U pip "poetry<1.8"
  poetry config virtualenvs.create false
  poetry install --with dev --with ci
 

.github/workflows/code-tree-analysis.yaml

@@ -17,7 +17,7 @@ jobs:
 
  - name: Install Python dependencies for deploying
  run: |-
- pip install -U pip poetry
+ pip install -U pip "poetry<1.8"
  poetry config virtualenvs.create false
  poetry install --with dev --with ci
 
@@ -46,4 +46,4 @@ jobs:
  uses: thollander/actions-comment-pull-request@v1
  with:
  message: "${{ steps.code-tree-analysis.outputs.pr-message }}"
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/lint.yaml

@@ -23,12 +23,12 @@ jobs:
 
  - name: Set up Poetry and upgrade pip
  run: |
- pip install -U pip poetry
+ pip install -U pip "poetry<1.8"
 
  - name: Install dependencies
  run: |
  poetry config virtualenvs.create false && poetry install --with dev --with ci
 
  - name: Lint with black, isort and flake8
  run: |
- task lint
+ task lint

.gitignore

@@ -1,7 +1,17 @@
-# user defined
+# Folders that Git should ignore
+*.ipynb_checkpoints/
+pipelines/precipitation_model/impa/models/data/
+pipelines/precipitation_model/impa/models/docs/
+pipelines/precipitation_model/impa/models/eval/
+pipelines/precipitation_model/impa/models/models/
+pipelines/precipitation_model/impa/models/personal/
+pipelines/precipitation_model/impa/models/radar_cal/
+pipelines/precipitation_model/impa/models/lightning_logs/
+**/test_flow/** */
+
+# User-defined
 .replit
 replit.nix
-**/data/
 test_local.py
 pylint.txt
 test.py
@@ -10,18 +20,68 @@ test/*.ipynb
 test/*.csv
 setup.py
 .vscode/*
-*.hdf
-
 
+# File extensions that Git should ignore
+
+# tex
+*.aux
+*.bbl
+*.blg
+*.fdb_latexmk
+*.fls
+*.out
+*.synctex.gz
+
+# images
+*.pdf
+*.png
+
+# data files
+*.csv
+*.xls
+*.pkl
+*.npy
+*.Rdata
+*.Rds
+*.hdf
+**/gypscie/**
 
-# Byte-compiled / optimized / DLL files
+# Python
+*.pyc
 __pycache__/
 *.py[cod]
 *$py.class
 
 # C extensions
 *.so
 
+# OS generated files
+.DS_Store
+.DS_Store?
+._*
+.Spotlight-V100
+.Trashes
+ehthumbs.db
+Thumbs.db
+
+# Compiled source
+*.com
+*.class
+*.dll
+*.exe
+*.o
+*.so
+
+# Packages
+*.7z
+*.dmg
+*.gz
+*.iso
+*.jar
+*.rar
+*.tar
+*.zip
+
 # Distribution / packaging
 .Python
 build/
@@ -43,6 +103,13 @@ share/python-wheels/
 *.egg
 MANIFEST
 
+# Logs and databases
+*.log
+*.sql
+*.sqlite
+pip-log.txt
+pip-delete-this-directory.txt
+
 # PyInstaller
 # Usually these files are written by a python script from a template
 # before PyInstaller builds the exe, so as to inject date/other infos into it.
@@ -52,6 +119,8 @@ MANIFEST
 # Installer logs
 pip-log.txt
 pip-delete-this-directory.txt
+*.manifest
+*.spec
 
 # Unit test / coverage reports
 htmlcov/
@@ -108,7 +177,7 @@ ipython_config.py
 # install all needed dependencies.
 #Pipfile.lock
 
-# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+# PEP 582
 __pypackages__/
 
 # Celery stuff
@@ -146,4 +215,14 @@ dmypy.json
 .pyre/
 
 # VSCode project settings
-.vscode/
+.vscode/
+
+# MLflow
+**/mlruns/**
+*.csv
+**/gypscie/processors/**
+!/gypscie/**/MLproject
+!/gypscie/**/etl_alertario.py
+!/gypscie/**/conda.yaml
+/gypscie/utils/logging.py
+!/gypscie/processors/utils/*

Dockerfile

@@ -4,9 +4,10 @@ ARG PYTHON_VERSION=3.10-slim
 # Start Python image
 FROM python:${PYTHON_VERSION}
 
-# Install git
+# Install git, gcc, build-essential, and other dependencies
+# hadolint ignore=DL3008
 RUN apt-get update && \
- apt-get install -y git && \
+ apt-get install -y --no-install-recommends git ffmpeg libsm6 libxext6 build-essential gcc && \
  apt-get clean && \
  rm -rf /var/lib/apt/lists/*
 

pipelines/constants.py

@@ -16,9 +16,11 @@ class constants(Enum):
  ######################################
  # Agent labels
  ######################################
- # EXAMPLE_AGENT_LABEL = "example_agent"
+ WEATHER_FORECAST_AGENT_LABEL = "weather-forecast"
 
  ######################################
  # Other constants
  ######################################
  # EXAMPLE_CONSTANT = "example_constant"
+ INFISICAL_USERNAME = "USERNAME"
+ INFISICAL_PASSWORD = "PASSWORD"

pipelines/flows.py

@@ -2,4 +2,4 @@
 """
 Imports all flows for every project so we can register all of them.
 """
-from pipelines.exemplo import * # noqa
+from pipelines.precipitation_model import * # noqa

pipelines/precipitation_model/__init__.py

@@ -0,0 +1,3 @@
+# -*- coding: utf-8 -*-
+from pipelines.precipitation_model.impa.flows import * # noqa
+from pipelines.precipitation_model.rionowcast.flows import * # noqa

pipelines/precipitation_model/impa/README.md

@@ -0,0 +1,134 @@
+# Rio Rain
+
+This project aims to provide a pipeline for real time prediction in Rio de Janeiro. It contains a script that automatically downloads the most recent satellite data to then process it and, finally, make predictions for the next 3 hours.
+
+## Table of Contents
+
+- [Rio Rain](#rio-rain)
+ - [Table of Contents](#table-of-contents)
+ - [Introduction](#introduction)
+ - [Features](#features)
+ - [Installation](#installation)
+ - [Usage](#usage)
+ - [Main script](#main-script)
+ - [Evaluation](#evaluation)
+ - [File structure](#file-structure)
+
+## Introduction
+
+This project contains the necessary code for nowcasting with 5 different models. It also includes a script that plots the predictions.
+
+## Features
+
+- Real-time prediction
+- Nowcasting visualization
+- Performance of each model in the last 3 hours
+
+## Installation
+
+To use the scripts contained in the provided zip file, the necessary libraries must first be correctly installed. There are different ways to do it:
+
+- Using `poetry`:
+ To install the necessary requirements using `poetry`, you may open the terminal on the root directory and call `poetry install`. Unfortunately, there is a necessary library that cannot be installed through `poetry`, so you can instead use `pip` for this particular library calling `pip install --no-use-pep517 mamba-ssm`.
+
+- Using `pip`:
+ To install the dependencies using `pip`, you can use the provided file `requirements.txt` by calling `pip install -r requirements.txt`.
+
+- Using other project managers:
+ You may also use other project managers like `conda` directly installing the libraries contained in `requirements.txt`.
+
+## Usage
+
+### Main script
+
+To produce predictions in real time, it is enough to call `python src/eval/update-real_time.py` on the root directory in the correct project environment. Once this command is called, the script will run the following tasks, in order:
+
+- Download GOES-16 data from AWS
+- Process the data locally
+- Build a dataset appropriate for making predictions
+- Make predictions for each of the provided models
+
+Once the script stops running, you can find the output files as described in the [file structure](#file-structure) section.
+
+Note that there are optional arguments for this command:
+
+`python src/eval/update-real_time.py [--cuda] [--num_workers] [--datetime]`,
+
+Here `--cuda` may be passed if you want to make predictions though GPU computing. If you want to do all calculations in CPU, you should not pass the optional argument `--cuda`. This will be slower and some models may not work in this mode.
+
+On the other hand `[--num_workers]` is the number of processes that may be run in parallel. It must be an integer value greater than zero. Generally, the larger this number is, the faster the script reaches its conclusion.
+
+Finally, `[--datetime]` may be passed to make predictions from the time passed in UTC. The format passed must be '%Y-%m-%d %H:%M:%S', so if we want predictions from 13/01/2024 14:00:00 BRT, we must call `bash src/eval/update-real_time.sh --datetime '2024-01-13 17:00:00'`.
+
+If it is desired to make predictions for just some of the models, it is possible to edit the file `src/eval/real_time_config.json` and delete the dictionary entries associated to the model that is to be excluded. Be mindful that the model `EVONET` is necessary for predicting with `NowcastNet`.
+
+### Evaluation
+
+Scripts for evaluating model predictions for the last three hours are also made available. Once the predictions are made through the main script, you may call `python src/eval/viz/plot_real_time.py [--num_workers]` to produce plots or `python src/eval/metrics/calc-metrics.py [--num_workers]` to calculate metrics.
+
+
+## File structure
+
+```
+rio-rain
+│ README.md
+│ poetry.lock
+│ pyproject.toml
+│
+└───data
+│ │
+│ └───dataframe_grids
+│ │ ...
+| dataframes
+| | ...
+| processed
+| | ...
+| raw
+| └ ...
+└───eval
+| └ ...
+|
+└───models
+| └ ...
+|
+└───predictions
+| └ ...
+|
+└───src
+ └ ...
+```
+
+In the `data` folder, the coordinate grids associated to the points in Earth's surface, raw data downloaded from AWS, locally processed data and dataframes ready for prediction are found in their respective subfolder.
+
+In the `eval` folder, the output metrics and prediction graphs are found in their respective subfolder.
+
+In the `models` folder, the parameters and necessary information associated to each model are found.
+
+In the `predictions` folder, the output predictions are saved in `.hdf` files which contain predictions for each model. In these files, the data is organized as follows:
+
+```
+root
+│
+└───date1
+│ │
+│ └───time1
+| |
+│ └───datetime1
+| |
+│ └───datetime2
+| |
+│ └─── ...
+| |
+| time2
+| | ...
+|
+└───date2
+| └ ...
+| ...
+```
+
+In this hierarchical structure, `date1`, `date2`, etc. refer to the date of the last observation informed to the model. `time1`, `time2`, etc. refer to the time of the last observation informed to the model. `datetime1`, `datetime2`, etc. represent the date and time of the prediction being made. All of these dates and times are in accordance with UTC timezone.
+
+In the leaf nodes given by `datetime1`, `datetime2`, etc. 256x256 matrices which represent the predictions are found. The coordinates of these pixels is in accordance with the `.npy` file found in `data/dataframe_grids/rio_de_janeiro-res=2km-256x256.npy`.
+
+Finally, we also have the `src` folder, in which all source code is found, relating to the downloading and processing of data, the scripts that load the saved models and make predictions, as well as the codes that evaluate the predictions.