diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 0000000..28f54bb --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,10 @@ +{ + "name": "Python 3.12 Development Container For DEEP", + "image": "python:3.12", + "extensions": [ + "ms-python.python", + "ms-python.vscode-pylance" + ], + "postCreateCommand": "pip install -e . && deepfacility ux", + "forwardPorts": [8000] +} diff --git a/.github/workflows/unittests.yml b/.github/workflows/unittests.yml new file mode 100644 index 0000000..6cbb500 --- /dev/null +++ b/.github/workflows/unittests.yml @@ -0,0 +1,29 @@ +name: tests + +on: + pull_request: + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + run-tests: + runs-on: ubuntu-latest + strategy: + max-parallel: 1 + matrix: + python-version: [ '3.12' ] + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages + run: | + pip install --upgrade pip + pip install --upgrade build + pip install -e .[test] + pip install -e .[i18n] + pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu118 + - name: Run unit-tests + run: pytest -v -m "not network_dependent" \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b80c62b --- /dev/null +++ b/.gitignore @@ -0,0 +1,39 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ + + +# Sphinx documentation +docs/_build/ + +# pyenv +.python-version + +# Environments +.env/ +.venv/ + +*.egg-info + +app-data/ +.idea/ +*venv*/ +build/ +from-research +config.toml +*.zip diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..c05c55e --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Bill & Melinda Gates Foundation + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..0dd7e27 --- /dev/null +++ b/README.md @@ -0,0 +1,92 @@ +# Digitally Enabled Equitably Placed (DEEP) Facility +Tools for optimizing placement of health workers and services based on village locations. + +## Prerequisites +- [Python](https://www.python.org/downloads) 3.12 or higher. +- [Python virtual environment](https://realpython.com/python-virtual-environments-a-primer/) + +## Setup +This section describes how to set up the tool and the demo web app. + +1. Activate your Python virtual environment and upgrade pip and build tools. +```bash +# Update pip and build package. +pip install --upgrade pip +pip install --upgrade build +pip install --upgrade setuptools +``` + +2. [Clone this repository](https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository) or [download](https://github.com/InstituteforDiseaseModeling/deepfacility/archive/refs/heads/main.zip) and extract the source code. Then open a terminal and navigate to the root directory of the source code. +```bash +# Navigate to the root directory. +cd deepfacility +``` + +3. Install the tool: +```bash +# Install the tool and required packages. +pip install -e . +``` + +## Getting Started + +Start the demo web app: +```bash +# Start the demo web app using `ux` command. +deepfacility ux +```` + +After you see the message `Serving on ...` the **demo** web app is running. + +Follow these steps to experience the workflow end-to-end: +- Open your web browser and navigate to [http://localhost:8000](http://localhost:8000) to access the **demo** web app. +- Follow instructions to "upload" and configure village locations .csv file. + - _Note that this app is running on your local machine and all files are stored locally._ +- Click `Prepare Data` to download and prepare input data: households and commune shapes. +- Select locations (communes) and click `Run Clustering` to start the processing. +- Visualize and explore village shapes and health facilities recommendations on a map. +- Obtain the results file. + +_Notes_: +- _This web app is only for **demo** purposes and is **not** intended for production use._ +- _To start a fresh copy of the app, without cached data, run `deepfacility reset` command before starting the app._ + +# Terminology +In this repository, we use the following terms and abbreviations: + +- `Cluster` and `Village` refer to the same entity with small differences in context: + - Cluster: a group of households. + - Village: spatial interpretation of a cluster. +- `Location`: an administrative area where the clustering is performed. + - In this tool a location value is a colon-separated list of names of administrative levels, per [configuration](docs/design.md#locations). + - For example, in Burkina Faso: + - `Tapoa:Diapaga` represents the `Diapaga` commune from the `Tapoa` province. + - `Tapoa:Diapaga:Mangou` represents a village from the `Diapaga` commune. + +- Abbreviations: + - Health Facilities (HF) + - Empirical Cumulative Distribution Function (eCDF) + +# Multi-Language Support +By default, the demo web app supports French and English languages. To add a support for additional languages see [Add New Language](docs/design.md#adding-new-languages) section in the design document. + +## Documentation +- [Components](docs/components.md) +- [Design](docs/design.md) +- [Scientific Workflow](docs/workflow.md) +- [CLI commands](docs/commands.md) + +## Data Sources +The system is using the following external data sources: +- Open Buildings (Google) + W. Sirko, S. Kashubin, M. Ritter, A. Annkah, Y.S.E. Bouchareb, Y. Dauphin, D. Keysers, M. Neumann, M. Cisse, J.A. Quinn. Continental-scale building detection from high resolution satellite imagery. arXiv:2107.12283, 2021. + +- [GADM shapes](https://gadm.org/data.html) for countries administrative areas + +## Package Dependencies +Packages listed in +[pyproject.toml](pyproject.toml) + +## Disclaimer +The code in this repository was developed by IDM to support our research into healthcare system capacity. We’ve made it publicly available under the MIT License to provide others with a better understanding of our research and an opportunity to build upon it for their own work. We make no representations that the code works as intended or that we will provide support, address issues that are found, or accept pull requests. You are welcome to create your own fork and modify the code to suit your own modeling needs as contemplated under the MIT License. + diff --git a/docs/commands.md b/docs/commands.md new file mode 100644 index 0000000..417450b --- /dev/null +++ b/docs/commands.md @@ -0,0 +1,79 @@ +# Command Line Tool Commands + +## Setup +Follow the [setup instruction](../README.md#setup) from the README. + +## Configuration +Confirm the tool has been installed and observe the help content: +```bash +# Check the tool has been installed and see the usage help. +deepfacility + +> usage: deepfacility run [-h] [-l LOCATION_FILTER [LOCATION_FILTER ...]] [-c CONFIG_FILE] [-n RUN_NAME] [-r RESULT_DIR] [--sid SESSION_ID] + {countries,config,prep,locations,run,viewmap,ux,reset} +``` + +See the list of supported countries: +```bash +deepfacility countries + +> INFO: Supported countries (you can set in config): +> ...list of countries... +``` + +Create a config file: +```bash +# Generate a config file at the default path: app-data/config.toml +deepfacility config +``` + +Update the config file to set paths and column names for your files: +- `[args.village_centers]` section for the village centers file. +- `[args.baseline_facilities]` section for the baseline facilities file (this is optional). + +## Data Preparation + +Prepare scientific workflow input files: +```bash +# Prepare input files +deepfacility prep +``` +The above command will: +- download and preprocess Google buildings and GADM shapes +- standardize your village centers and baseline facilities files + +## Scientific Workflow + +See the list of all available locations execute the `locations` command: +```bash +# Read all locations available in the input data. +deepfacility locations +``` +_Note: The filter can be a specific location or a location [regex](https://docs.python.org/3/howto/regex.html#regex-howto) pattern._ + +Process specified subset of locations. +```bash +# Run the processing for a subset of locations. +deepfacility run -l "Noumbiel:.*" # All locations in the Noumbiel province +``` + +To process all locations execute the `run` command without a location filter (this may take 1-2h). + +## Visualizing Results +Generate the interactive visualization map by specifying the result directory: +```bash +# Create viz map for results generated with -l `Noumbiel:.*` +deepfacility viewmap -r Noumbiel-_1_5a819a9 +```` +_Note: Look at the `app-data/data/BFA/results` directory for the list of available results directories._ + +## Testing +Install the test dependencies: +```bash +# Install test dependencies. +pip install -e .[test] +``` +Run available tests: +```bash +pytest -v +``` diff --git a/docs/components.md b/docs/components.md new file mode 100644 index 0000000..8c414ef --- /dev/null +++ b/docs/components.md @@ -0,0 +1,235 @@ +# Solution Components +This document provides a high-level overview of solution components. + +## Table of Contents +* [Code Structure](#code-structure) +* [Components Diagram](#components-diagram) +* [Workflows](#workflows) +* [User Experience (UX)](#user-interface-ux) +* [Command Line Tool (CLI)](#command-line-tool) + + +## Code Structure + +The code structure is organized into modules and packages. The main modules are: +- `deepfacility.flows`: Workflow classes which methods drive data preparation and scientific workflows. +- `deepfacility.data`: data preparation workflow steps. +- `deepfacility.tasks`: scientific workflow functionality. +- `deepfacility.viz`: visualization and interactive map features. +- `deepfacility.config`: configuration classes. +- `deepfacility.utils`: utility functions. +- `deepfacility.lang`: language translation features. +- `deepfacility.ux`: demo web UI app. + +## Components Diagram + +```mermaid +%%{ + init: { + 'theme': 'base', + 'themeVariables': { + 'primaryColor': '#eeffcfff', + 'primaryTextColor': 'black', + 'primaryBorderColor': 'black', + 'lineColor': '#789abc', + 'secondaryColor': '#006100', + 'tertiaryColor': '#ffffff', + 'tertiaryBorderColor': 'lightgray' + } + } +}%% +graph TD; + subgraph Workflows + external_data((External Data + Google Open buildings + GADM country admin shapes)) --> + data_prep(Data Preparation + Download external data + Unify user data per config + Match user data to admin names + Add metadata to support viz); + user_data((User Data + Village centers + Baseline health facilities)) --> + data_prep --> + ready_inputs((Inputs + Admin shapes and locations + Building coordinates + Village center coordinates + Baseline facilities coordinates)) --> + scientific_workflow(Scientific Workflow + Cluster households into villages + Create village shapes + Calculate facility/household distances + Recommend health facility coordinates) --> + result_files((Results + Locations + Clustered households + Village shapes + Placement recommendations + Population coverage plots + Logs)); + data_prep <--> + data_cache((Data Cache + Function calls caching + using joblib.Memory objects + and a local cache directory.)); + scientific_workflow <--> data_cache; + config((Configuration + User provided files + Workflow params and file paths)) --> data_prep; + config --> scientific_workflow; + end + subgraph "App(UX)" + html_templates(HTML Templates + UI declared using HTMX) --> + fast_api(FastAPI Backend + Handles web requests to render HTML templates into web UI. + Receives user inputs and files to configure and run workflows.); + css(CSS + HTML style definitions.) --> fast_api; + + fast_api --> + background_tasks(Background Tasks + Run workflows in background. + Progress is reported in UI via logs.); + background_tasks --> data_prep; + background_tasks --> scientific_workflow; + + fast_api <--> + rendered_ui(Web UI + Users interact with backend to configure + and run workflows and visualize results.); + + fast_api <--> session(Session + Functionality to configure and run workflows. + Preserving user session state between requests, + like configuration and language selection.); + session <--> config; + + fast_api <--> multi_lang_support(Multi-Language Support + Functionality for translating UI and backend messages. + Either a text dictionary or a ML language model.); + + rendered_ui --> viz_map(Visualization Map + Leaflet UI overlaying workflow results on an interactive map.); + result_files --> fast_api; + end +``` +Notes: +- The diagram uses terms and abbreviations describes in the [Terminology](../README.md#terminology) section in the main README. +- For simplicity, the CLI part of the components is not included in the diagram. + +--- +## Workflows +The core library is responsible for implementing the data and scientific workflows and provide supporting functionality like configuration, multi-language support, data caching. + +The processing workflows are driven from the [flows.py](../src/deepfacility/flows.py) module which is a great starting point for following the code flow: +- `prepare_inputs` - this function drives the data preparation workflow +- `process_locations` - this function drives the scientific workflow + +These two functions are driving two main workflows: +- [Data preparation](#data-preparation-workflow) workflow's code is in the [data](../src/deepfacility/data) directory: + - [download.py](../src/deepfacility/data/downloads.py) - functions for downloading external data + - [inputs.py](../src/deepfacility/data/inputs.py) - functions for preparing input files + +- [Scientific workflow](#scientific-workflow) workflow's code is in the [tasks](../src/deepfacility/tasks) directory: + - [clustering.py](../src/deepfacility/tasks/clustering.py) - functions for clustering households + - [distance.py](../src/deepfacility/tasks/distance.py) - functions for calculating distances + - [outlines.py](../src/deepfacility/tasks/outlines.py) - functions for creating village shapes + - [placement.py](../src/deepfacility/tasks/placement.py) - functions for generating recommendations for health facilities locations + +### Data Preparation Workflow +The data preparation workflow is responsible for preparing input data for the scientific workflow, which includes the following steps: +- Download Google Open Buildings for the specified country using [config/countries_s2_tokens.json](../src/deepfacility/config/countries_s2_tokens.json) file to determine the list of s2 geometry cells to download. +- Download [GADM](https://gadm.org/download_country.html) shapes for the specified country. +- Clip Google Open Buildings to GADM country shapes and produce a households file containing admin names (from GADM shapes) and building coordinates. +- Standardize user provided village centers file by associating admin names from GADM shapes through a spatial join between center coordinates and admin shapes. +- Optionally, standardize user provided baseline health facilities file (in the same way as village centers) add enrich it with metadata to support visualizations. + +In summary, the data preparation workflow ensures the scientific workflow receives standardized input files, +consistent with the system config and GADM admin names. + +### Scientific Workflow +The processing workflow receives the input files produced by the data preparation workflow and executes +the scientific workflow to produce the recommended health facility coordinates and population coverage metrics. + +Here is a high-level overview of the scientific workflow code structure: +- [flow.py](../src/deepfacility/flows.py) - contains top level workflow functions. + - [tasks/clustering.py](../src/deepfacility/tasks/clustering.py) - contains functions for clustering households. + - [tasks/outlines.py](../src/deepfacility/tasks/outlines.py) - contains functions for creating village shapes and merging results. + - [tasks/placement.py](../src/deepfacility/tasks/placement.py) - contains functions for recommending health facility placements. + +--- + +## User Interface (UX) + +The UX implemented in the `deepfacility` package was designed for demo purposes. +It provides a web-based interface for users to configure the tool, run the data preparation and scientific workflows and +generate and explore visualizations. + +### UX Tech Stack + +The technology stack used in this UX solution for the main web app scenarios includes [HTMX](https://htmx.org/) and [FastAPI](https://fastapi.tiangolo.com/) Python packages. + +The [HTMX](https://htmx.org/) package is used to provide a seamless user experience by updating parts of the page without a full page reload. +This solution also removes JavaScript from the front-end HTML templates which simplifies the solution development. +Still, under-the-hood when [HTMX templates](../src/deepfacility/ux/templates) are rendered they still contain JavaScript. + +On the backend side, [FastAPI](https://fastapi.tiangolo.com/) is used to provide a REST API for the web app in the [ux/main](../src/deepfacility/ux/main.py) module. + +The [session](../src/deepfacility/ux/session.py) module implements the session object which is: +- Instantiated when the first request is processed. +- Stored in the app state and is available to all requests. +- Has a reference to a config which is used throughout the app for constructing file paths. + +### Configuration UI +The app configuration allows users to upload the village centers and baseline health facilities (optional) files and +select which columns contain coordinates and which represent village names. + +The backend then uses geopandas and pycountry packages to determine the country (and country code) based on the coordinates of the village centers. + +After this, a config file is created in the session dir and its reference is stored in the session object. + +Finally, the user is shown configuration parameters and a preview of the content of uploaded files. +If all looks good, the user can proceed and kick off the data preparation workflow. + +In case the app is stopped and the session object needs to be recreated, the app will look in the session directory for +the latest saved config file and use it to recreate the session object. + +### Data Preparation UI +After the `Prepare Input Files` button is clicked, the data preparation workflow is started as a background process. +UI uses the approach described in the [Monitoring Progress UI](design.md#monitoring-progress-ui) section to shows the progress in the form of +stage completion and background task logs. + +The artifacts of the data preparation workflow are stored in the `inputs` dir and they include: +- locations csv file, containing a list of available admin locations +- admin shapes geojson file, containing specified admin level shapes +- households csv file, containing country building coordinates and admin names +- village centers csv file, containing standardized village centers data +- baseline facilities csv file, containing standardized baseline facilities data + +### Scientific Workflow UI +After input files are ready a user can use the scientific workflow UI to select locations and run the scientific workflow. +The scientific workflow is started as a background process and the progress is reported in the UI. +This scientific workflow UI and backend work in the similar way as the data preparation UI and backend. + +### Results Visualization UI +After the scientific workflow is completed, the visualization map is loaded automatically. +The web visualization component is providing an interactive map of health facilities and villages, +is implemented in html/javascript solution using the [Leaflet](https://leafletjs.com/) javascript library. +All the result files used for visualization are also available for download. +The UI also offers ability to select and see visualizations of previous workflow runs. + +--- +## Command Line Tool +The command line interface (CLI) supports a set of commands which allow users to: +- see available countries and locations +- configure the tool +- run the data preparation workflow +- run the scientific workflow +- generate web visualization map +- run the UX web app + +In other words, CLI support all the functionality offers via UX. + diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..8dd09fd --- /dev/null +++ b/docs/design.md @@ -0,0 +1,351 @@ +# Design Concepts +This document describes main design concepts and implementation details of the `deepfacility` tool. + +## Table of Contents +* [Environment Variables](#environment-variables) +* [Configuration](#configuration) +* [Location Format](#location-format) +* [UX Sessions](#ux-sessions) +* [UX Background Tasks](#ux-background-tasks) +* [Multi-Language Support](#multi-language-support) +* [Logging](#logging) +* [Parallelization](#parallelization) +* [Data Caching](#data-caching) +* [Other Considerations and Limitations](#other-considerations-and-limitations) + +## Environment Variables +The following environment variables are used in the tool: + +| Environment Variable | Default | Description | +| ---|-------------|---| +| DEEPFACILITY_ROOT_DIR | `app-data` | The root directory of the app. | +| DEEPFACILITY_LANG_MODEL | `NLP` | Set the translation ML model to use (default isHelsinki-NLP/opus-mt-en-fr). | +| DEEPFACILITY_HOST | `localhost` | Demo web app host name. | +| DEEPFACILITY_PORT | `8000` | Demo web app port. | +| DEEPFACILITY_SID | `None` | Set the session id for the CLI scenario. | + + +## Configuration +The configuration consists of user and system configuration TOML files. + +### Configuration Templates +The user configuration is focused on files a user needs to provide and describe required column names. +The system configuration describe workflow input and results files. + +### Configuration Template Variables +During the initiation, the variables are replaced with values collected from users or determined based on user inputs: +- `{app_dir}`: App root directory containing the cache, data, and downloads directories and the config file. +The default is `app-data`, positioned relative to the working directory, for example, the repo dir. +It can be customized using the `DEEPFACILITY_ROOT_DIR` environment variable. +- `{data_dir}`: Data directory containing country directories with input and results files. +- `{country_code}`: Country code determined based on the village centers coordinates. +- `{level}`: Admin level determined based on the village centers coordinates. + +The following variables are replaced with values determined at runtime: +- `{run_name}`: A unique run name based on the selected locations. +- `{location}`: The location iterator value. + +### Configuration Initiation +The configuration initiation is done in the `deepfacility.config` module. The `Config` class is used to load, populate and merge the user and system configuration files. +In the UX scenario, users are not directly exposed to configuration files. Instead, the session object, from the `deepfacility.ux` module, generates the configuration based of user inputs and uploaded files. +In the CLI scenario, the user can create a configuration file using the `config` command. Then the user can populate the configuration file to match files and columns in the data. + +### Accessing Configuration +Methods of WorkflowEntity subclasses have access to the config through the `cfg` field. Other function receive the configuration object as an argument. + +### Configuration Usage +The configuration contains most of the information needed for the tool to run. +This means that most functions could receive the configuration object as the only argument. +However, this approach would make the code less modular and harder to test. + +Therefore, the general principle is for functions to have input values as explicit arguments and use parameters and +file paths from the configuration object for generate outputs. This allows the code to be tested independently of the configuration object. + +### Configuration File Structure +The configuration file is a TOML file which contains sections for user and system configurations. +The configuration file contains the following sections: +- `args`: basic parameters like country, data directory, thresholds and file path and column names for user-provided files. +- `downloads`: contains URLs for downloading external data, directories for storing downloaded files and coordinates column names. +- `inputs`: contains file paths and column names for input files generated by the data preparation workflow. +- `results`: contains file paths and column names for output files generated by the scientific workflow. + +For more detail see configuration templates which contain detailed descriptions of all configuration parameters: +- [user config template](../src/deepfacility/config/template_user.toml) +- [system config template](../src/deepfacility/config/template_sys.toml). + +### Directory Structure +The configuration file facilitates the directory structure of the app. + +Typical structure of the root `app-data` dir used by the web app looks like this: +```bash +app-data +│ +├──── downloads # all downloads for all countries +│ ├── GADM_shapes # all GADM shapes for all countries +│ │ └── gadm41_BFA_shp.zip +│ └── google_buildings # all Google Open Buildings for all countries +│ ├── 0e3_buildings.csv.gz +│ ├── 0e5_buildings.csv.gz +│ ├── 0fb_buildings.csv.gz +│ ├── 0fd_buildings.csv.gz +│ └── 11d_buildings.csv.gz +│ +├── 9593161cb53f # session id +│ ├── config.village_centers.toml # generated user and system configuration file +│ └── data # all data for all countries│ +│ └── BFA # all data for a specific country +│ ├── args # args: user provided data +│ │ ├── health_facilities.csv +│ │ └── locality_villages.csv +│ │ +│ ├── inputs # inputs: data generated by the data preparation workflow +│ │ ├── all_locations.csv # all available admin locations +│ │ ├── baseline_facilities.csv # baseline health facilities with matched admin names +│ │ ├── baseline_facilities.geojson # baseline health facilities points for visualization +│ │ ├── buildings_BFA.feather # Google Open Buildings clipped for BFA +│ │ ├── households.csv # households coordinates with matched admin names +│ │ ├── households.stats.csv # households stats +│ │ ├── prep.log # data preparation workflow log +│ │ ├── shapes # GADM shapes +│ │ ├── village_centers.csv # village centers with matched admin names +│ │ └── village_centers.geojson # village centers points for visualization +│ │ +│ └── results # results: data generated by the scientific workflow +│ └── Bale-Boromo_3_41353ad # results for 3 communes (Bale-Boromo and two other) +│ ├── cluster_centers.csv # cluster centers +│ ├── cluster_counts.csv # cluster counts +│ ├── cluster_stats.csv # cluster stats +│ ├── clustered_households.csv # clustered households +│ ├── locations.csv # locations +│ ├── optimal_facilities.csv # optimal health facilities +│ ├── population_coverage_optimal.png # optimal population coverage plot +│ ├── population_coverage_baseline.png # existing population coverage plot +│ ├── run.log # scientific workflow log +│ ├── village_shapes.geojson # village shapes for visualization +│ └── www # interactive visualization map +│ +└── cache # cache directory managed by the system (joblib.Memory python package) + └── deepfacility # cache directory for the deepfacility package + ├── data # data preparation functions cache + ├── tasks # scientific workflow functions cache + └── utils # utility functions cache +``` + +## Location Format +As described in the main readme a location is an administrative area where the clustering is performed. +Location values are constructed from colon-separated names of administrative levels. + +The default configuration for Burkina Faso specifies the following location formats: + +- Communes: `{province}:{commune}` (e.g., `Tapoa:Diapaga`) + - Data from GADM shapes (columns NAME2, NAME3) +- Villages: `{province}:{commune}:{village}` (e.g., `Tapoa:Diapaga:Mangou`) + - Data from user-provided village centers file (custom column) + +Location columns are specified in the configuration: + - Commune column: [system configuration](../src/deepfacility/config/template_sys.toml) > `[inputs.shapes].adm_cols` + - Village column: [user configuration](../src/deepfacility/config/template_user.toml) > `[args.village_centers].adm_cols` + +## UX Sessions +The main purpose of the session object is to preserve references to a config and translator objects between requests +and to facilitate the execution of the background tasks. + +UX session support is implemented using the `deepfacility.ux.session` module, with `deepfacility.ux.session.init` +being the function which handles most of the session management: creation and retrieval. + +The session object is created when the user starts the web app. At that time a session id is generated and used to store +the reference to a session object in the session dictionary. +The session dictionary is created in the FastAPI state object and preserved until the app is stopped. + +The session id is also stored as long-living cookie in the user's browser. The session id is used to retrieve the session object from the session dictionary when the user makes a new request. + +In case a session id cookie is lost a user would have to start a new session. This would also mean loosing all the previously generated files. +To mitigate that, a user has few options: +- Set the session id in the URL: `http:://localhost:8000?sid=1234` +- Set the session id in the environment variable: `export DEEPFACILITY_SID=1234` before running the `deepfacility ux` command. +```bash +# Set the session id with the environment variable +export DEEPFACILITY_SID=1234 +deepfacility ux +``` + +- Use the CLI scenario where the session id is set as a command line argument. +```bash +# Hardcode the session id for all session +deepfacility ux --sid 1234 +``` + + +## UX Background Tasks +The background tasks are run using FastAPI `BackgroundTasks` object to asynchronously run the data preparation and scientific workflows commands in the background. + +### Execution Control +The background task execution is abstracted using `config.Operation` and `ux.Session` classes. + +The `Operation` class fields are used to control workflow functions execution: +- `conrol_file` - a file used to signal the workflow function which is running in the background tasks to stop. +- `log_file` - point to a log file where workflow function logs are stored (and from where the logs are streamed to UI). +- `logger` - a logger object used to log messages to the log file. + +The `Operation` is an abstract class inherited by `Inputs` and `Results` config classes which are used by data preparation and scientific workflows functions. + +The `ux.Session` class has a private member `_operation` which points to a workflow operation object when a user triggers execution. +It also has `start_task` and `stop_task` methods meant to facilitate the execution of the background tasks. + +### Monitoring Progress UI +This section describes how workflow execution progress is monitored in the UI. + +The steps below are for the data preparation workflow. The scientific workflow monitoring uses the same approach. + +1. User initiates data preparation: + - User clicks the "Prepare Input Data" button on the data prep page [30-prep.html](../src/deepfacility/ux/templates/30-prep.html). + - This triggers the `ux.main.prep` function which: + - Starts data preparation background tasks. + - Renders the status monitoring page [30-prep-status-container.html](../src/deepfacility/ux/templates/30-prep-status-container.html). + +2. Status monitoring: + - The status monitoring page automatically sends `/prep/status` request every 5 seconds. + - This is specified with htmx `hx-trigger` attribute as shown in the example below. + - The `ux.main.prep_status` function on the server: + - Checks the status of input files and backend tasks. + - Renders a status update response based on the progress. + - The response is displayed in the `status` div in the page [30-prep-status.html](../src/deepfacility/ux/templates/30-prep-status.html). + ```html +
Waiting for status...
+
+
+ + + + |
+ + + | +
+ + + + + | +|
+ + | +
{{ _("Waiting for status...") }}
+
+
|
+ + |
+ {% for line in logs %}
+ {{ _(line) }} + {% endfor %} + |
+
+
+ + + {% if baseline_file %} + + {% endif %} + |
+
+ + {{ village_preview|safe }} + |
+
+ + {{ baseline_preview|safe }} + |
+
{{ _("Waiting for status...") }}
+
+
|
+ + |
+ {% for line in res_logs %}
+ {{ _(line) }} + {% endfor %} + |
+
+ {{ _("Digitally Enabled Equitably Placed (DEEP) Facility") }}+ |
+
+
+
+ {{ _("Warning: Changing the language will not update the map created previously.") }}
+
+ |
+
+ + | ++ + | ++ + | ++ + | +
(e.x-g.x)*(h.y-g.y)?(1===c&&b.lineTo(g.x,g.y),c=0,l||b.moveTo(g.x,g.y),b.lineTo(h.x,h.y)):(0===c&&b.lineTo(e.x,e.y),c=1,l||b.moveTo(e.x,e.y),b.lineTo(m.x,m.y));b.closePath();b.fill()}},w={draw:function(b,a,c,d,f,e,g,h,k){a={x:a.x-p,y:a.y-n};var m=q/(q-f),l=q/(q-e);f=r.project(a,m);d*=m;e&&(a=r.project(a,l),c*=l);(m=this._tangents(a, +c,f,d))?(e=P(m[0].y1-a.y,m[0].x1-a.x),m=P(m[1].y1-a.y,m[1].x1-a.x)):(e=1.5*E,m=1.5*E);b.fillStyle=g;b.beginPath();b.arc(f.x,f.y,d,J,e,!0);b.arc(a.x,a.y,c,e,J);b.closePath();b.fill();b.fillStyle=h;b.beginPath();b.arc(f.x,f.y,d,m,J,!0);b.arc(a.x,a.y,c,J,m);b.closePath();b.fill();b.fillStyle=k;this._circle(b,f,d)},simplified:function(b,a,c){this._circle(b,{x:a.x-p,y:a.y-n},c)},shadow:function(b,a,c,d,f,e){a={x:a.x-p,y:a.y-n};f=z.project(a,f);var g;e&&(a=z.project(a,e));var h=this._tangents(a,c,f,d); +h?(e=P(h[0].y1-a.y,h[0].x1-a.x),g=P(h[1].y1-a.y,h[1].x1-a.x),b.moveTo(h[1].x2,h[1].y2),b.arc(f.x,f.y,d,g,e),b.arc(a.x,a.y,c,e,g)):(b.moveTo(a.x+c,a.y),b.arc(a.x,a.y,c,0,2*E))},shadowMask:function(b,a,c){var d=a.x-p;a=a.y-n;b.moveTo(d+c,a);b.arc(d,a,c,0,2*E)},hitArea:function(b,a,c,d,f,e,g){a={x:a.x-p,y:a.y-n};var h=q/(q-f),k=q/(q-e);f=r.project(a,h);d*=h;e&&(a=r.project(a,k),c*=k);e=this._tangents(a,c,f,d);b.fillStyle=g;b.beginPath();e?(g=P(e[0].y1-a.y,e[0].x1-a.x),h=P(e[1].y1-a.y,e[1].x1-a.x),b.moveTo(e[1].x2, +e[1].y2),b.arc(f.x,f.y,d,h,g),b.arc(a.x,a.y,c,g,h)):(b.moveTo(a.x+c,a.y),b.arc(a.x,a.y,c,0,2*E));b.closePath();b.fill()},_circle:function(b,a,c){b.beginPath();b.arc(a.x,a.y,c,0,2*E);b.stroke();b.fill()},_tangents:function(b,a,c,d){var f=b.x-c.x,e=b.y-c.y,g=a-d,h=f*f+e*e;if(!(h<=g*g)){var h=pa(h),f=-f/h,e=-e/h,g=g/h,h=[],k,m,l;k=pa(K(0,1-g*g));for(var n=1;-1<=n;n-=2)m=f*g-n*k*e,l=e*g+n*k*f,h.push({x1:b.x+a*m<<0,y1:b.y+a*l<<0,x2:c.x+d*m<<0,y2:c.y+d*l<<0});return h}}},R={draw:function(b,a,c,d,f,e,g){var h= +q/(q-f);c=r.project({x:c.x-p,y:c.y-n},q/(q-d));d={x:0,y:0};for(var k={x:0,y:0},m=0,l=a.length-3;m(c.x-d.x)*(k.y-d.y)&&(b.fillStyle=d.x k.x&&d.y>k.y?g:e,b.beginPath(),this._triangle(b,d,k,c),b.closePath(),b.fill())},_triangle:function(b,a,c,d){b.moveTo(a.x,a.y);b.lineTo(c.x,c.y);b.lineTo(d.x,d.y)},_ring:function(b,a){b.moveTo(a[0]-p,a[1]-n);for(var c=2,d=a.length-1;c< +d;c+=2)b.lineTo(a[c]-p,a[c+1]-n)},shadow:function(b,a,c,d,f){var e={x:0,y:0},g={x:0,y:0};c=z.project({x:c.x-p,y:c.y-n},d);d=0;for(var h=a.length-3;d (c.x-e.x)*(g.y-e.y)&&this._triangle(b,e,g,c)},shadowMask:function(b,a){this._ring(b,a)},hitArea:function(b,a,c,d,f,e){var g=q/(q-f);c=r.project({x:c.x-p,y:c.y-n},q/(q-d));d={x:0,y:0};var h={x:0,y:0};b.fillStyle=e;b.beginPath();e=0;for(var k= +a.length-3;e (c.x-d.x)*(h.y-d.y)&&this._triangle(b,d,h,c);b.closePath();b.fill()}},r={project:function(b,a){return{x:(b.x-M)*a+M<<0,y:(b.y-N)*a+N<<0}},render:function(){var b=this.context;b.clearRect(0,0,B,v);if(!(x a.scale?a.height*a.scale:a.height;d=0;a.minHeight&&(d=1>a.scale?a.minHeight*a.scale:a.minHeight);g=a.wallColor||ha;h=a.altColor||aa;k=a.roofColor||X;b.strokeStyle=h;switch(a.shape){case "cylinder":w.draw(b,a.center,a.radius,a.radius,c,d,g,h,k);break;case "cone":w.draw(b,a.center,a.radius,0,c,d,g,h);break;case "dome":w.draw(b,a.center,a.radius,a.radius/2,c,d,g,h);break;case "sphere":w.draw(b,a.center,a.radius,a.radius,c,d,g,h,k);break;case "pyramid":R.draw(b, +e,a.center,c,d,g,h);break;default:Z.draw(b,e,a.holes,c,d,g,h,k)}switch(a.roofShape){case "cone":w.draw(b,a.center,a.radius,0,c+a.roofHeight,c,k,""+I.parse(k).lightness(0.9));break;case "dome":w.draw(b,a.center,a.radius,a.radius/2,c+a.roofHeight,c,k,""+I.parse(k).lightness(0.9));break;case "pyramid":R.draw(b,e,a.center,c+a.roofHeight,c,k,I.parse(k).lightness(0.9))}}}}},ia={maxZoom:G+2,maxHeight:5,isSimple:function(b){return x<=this.maxZoom&&b.height+b.roofHeight this.maxZoom))for(var a,c,d=F.items,f=0,e=d.length;f =this.maxHeight)&&(c=a.footprint,V(c)))switch(b.strokeStyle=a.altColor||aa,b.fillStyle=a.roofColor||X,a.shape){case "cylinder":case "cone":case "dome":case "sphere":w.simplified(b,a.center,a.radius);break;default:Z.simplified(b,c,a.holes)}}},z={enabled:!0,color:"#666666",blurColor:"#000000",blurSize:15,date:new Date,direction:{x:0,y:0},project:function(b,a){return{x:b.x+ +this.direction.x*a,y:b.y+this.direction.y*a}},render:function(){var b=this.context,a,c,d;b.clearRect(0,0,B,v);if(!(!this.enabled||x =a.altitude))){c=1/ka(a.altitude);d=5>c?0.75:1/c*5;this.direction.x=Fa(a.azimuth)*c;this.direction.y=Ea(a.azimuth)*c;var f,e,g,h;a=F.items;b.canvas.style.opacity=d/(2*C);b.shadowColor=this.blurColor;b.shadowBlur=C/2*this.blurSize;b.fillStyle=this.color;b.beginPath();d=0;for(c=a.length;d f.scale?f.height*f.scale:f.height;g=0;f.minHeight&&(g=1>f.scale?f.minHeight*f.scale:f.minHeight);switch(f.shape){case "cylinder":w.shadow(b,f.center,f.radius,f.radius,e,g);break;case "cone":w.shadow(b,f.center,f.radius,0,e,g);break;case "dome":w.shadow(b,f.center,f.radius,f.radius/2,e,g);break;case "sphere":w.shadow(b,f.center,f.radius,f.radius,e,g);break;case "pyramid":R.shadow(b,h,f.center,e,g);break;default:Z.shadow(b,h,f.holes,e,g)}switch(f.roofShape){case "cone":w.shadow(b, +f.center,f.radius,0,e+f.roofHeight,e);break;case "dome":w.shadow(b,f.center,f.radius,f.radius/2,e+f.roofHeight,e);break;case "pyramid":R.shadow(b,h,f.center,e+f.roofHeight,e)}}b.closePath();b.fill();b.shadowBlur=null;b.globalCompositeOperation="destination-out";b.beginPath();d=0;for(c=a.length;d >8&255,a>>16&255].join()+")"}}, +$,A={container:document.createElement("DIV"),items:[],init:function(){this.container.style.pointerEvents="none";this.container.style.position="absolute";this.container.style.left=0;this.container.style.top=0;z.context=this.createContext(this.container);ia.context=this.createContext(this.container);r.context=this.createContext(this.container);Y.context=this.createContext()},render:function(b){Ga(function(){b||(z.render(),ia.render(),Y.render());r.render()})},createContext:function(b){var a=document.createElement("CANVAS"); +a.style.transform="translate3d(0, 0, 0)";a.style.imageRendering="optimizeSpeed";a.style.position="absolute";a.style.left=0;a.style.top=0;var c=a.getContext("2d");c.lineCap="round";c.lineJoin="round";c.lineWidth=1;c.imageSmoothingEnabled=!1;this.items.push(a);b&&b.appendChild(a);return c},appendTo:function(b){b.appendChild(this.container)},remove:function(){this.container.parentNode.removeChild(this.container)},setSize:function(b,a){for(var c=0,d=this.items.length;c OSM Buildings