From eea3a514dff9b44fc2f8fd4269b2df2a81d33377 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 24 Jul 2024 12:38:26 -0600 Subject: [PATCH 1/3] Add Docker build config and continuous delivery --- .github/workflows/build-and-publish.yml | 54 +++++++++++++++++++++++++ Dockerfile | 26 ++++++++++++ compose.yml | 11 +++++ doc/operation.md | 10 +++++ 4 files changed, 101 insertions(+) create mode 100644 .github/workflows/build-and-publish.yml create mode 100644 Dockerfile create mode 100644 compose.yml diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml new file mode 100644 index 0000000..dccba06 --- /dev/null +++ b/.github/workflows/build-and-publish.yml @@ -0,0 +1,54 @@ +name: "Build & publish artifacts" + +on: + push: + branches: + - "main" + - "v2" # TODO: Remove before merge! + tags: + - "v[0-9]+.[0-9]+.[0-9]+*" + +concurrency: + group: "${{ github.workflow }}-${{ github.ref }}" + cancel-in-progress: true + +env: + # Many color libraries just need this to be set to any value, but at least + # one distinguishes color depth, where "3" -> "256-bit color". + FORCE_COLOR: 3 + +jobs: + # build-python-package: + # name: Build Python package + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # with: + # fetch-depth: 0 + # - uses: hynek/build-and-inspect-python-package@v2 + + # publish-python-package: + # name: Publish Python package to PyPI + # if: github.ref_type == 'tag' + # needs: ["build-python-package"] + # runs-on: ubuntu-latest + # environment: pypi + # permissions: + # id-token: write + # steps: + # - uses: actions/download-artifact@v4 + # with: + # name: Packages + # path: dist + + # - uses: pypa/gh-action-pypi-publish@release/v1 + # if: github.event_name == 'release' && github.event.action == 'published' + # with: + # # Remember to tell (test-)pypi about this repo before publishing + # # Remove this line to publish to PyPI + # repository-url: https://test.pypi.org/legacy/ + + build-and-publish-docker-image: + name: "Build and publish Docker image" + uses: "nsidc/.github/.github/workflows/build-and-publish-container-image.yml@main" + secrets: "inherit" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c68768d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,26 @@ +FROM mambaorg/micromamba:1.5.8-alpine3.20 + +WORKDIR /app + +# Activate the conda environment during build process +ARG MAMBA_DOCKERFILE_ACTIVATE=1 + +COPY ./conda-lock.yml . + +# NOTE: `-p` is important to install to the "base" env +RUN micromamba install -y \ + -p /opt/conda \ + -f conda-lock.yml \ + && micromamba clean --all --yes + + +# Install source +COPY ./pyproject.toml . +COPY ./antarctica_today ./antarctica_today +COPY ./qgis ./qgis + +# TODO: `pip install --editable .` +ENV PYTHONPATH=. + +# Extend the default micromamba entrypoint to use our CLI +ENTRYPOINT ["/usr/local/bin/_entrypoint.sh", "python", "antarctica_today"] diff --git a/compose.yml b/compose.yml new file mode 100644 index 0000000..82d0949 --- /dev/null +++ b/compose.yml @@ -0,0 +1,11 @@ +services: + cli: + image: "nsidc/antarctica_today:local" + build: "." + user: "root" + volumes: + - "./Tb/:/app/Tb/" + - "./data/:/app/data/" + - "./database/:/app/database/" + - "./plots/:/app/plots/" + - "./baseline_datasets/:/app/baseline_datasets/:ro" diff --git a/doc/operation.md b/doc/operation.md index 805f82c..683511e 100644 --- a/doc/operation.md +++ b/doc/operation.md @@ -186,3 +186,13 @@ python antarctica_today process - [ ] After this step, `git status` shows changed files. Should they be committed? + + +## Running in Docker + +This repository includes a `compose.yml` configuration which enables running this code +with Docker. For example, the `download-tb` command can be run as follows: + +```bash +docker compose run cli download-tb +``` From 2784faf1a07f94792f289b416a910965e50011c1 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 24 Jul 2024 12:57:25 -0600 Subject: [PATCH 2/3] Re-organize ops doc notes for consistency --- doc/operation.md | 63 ++++++++++++++++++++++++++++++------------------ 1 file changed, 39 insertions(+), 24 deletions(-) diff --git a/doc/operation.md b/doc/operation.md index 683511e..7e31b43 100644 --- a/doc/operation.md +++ b/doc/operation.md @@ -39,14 +39,14 @@ threshold binaries provided by Tom Mote. These are checked in to this repository ## 1. Download NSIDC-0080 -Download NSIDC-0080 granules into the `Tb/` directory: +Download NSIDC-0080 granules: ```bash PYTHONPATH=. python antarctica_today download-tb ``` -> [!IMPORTANT] +> [!NOTE] > For data before 2022-01-10, data have already been processed through step 2 and are > available as binary ".bin" files in this repo's `/data/daily_melt_bin_files` > directory. This data was generated from `NSIDC-0001` and `NSIDC-0007` datasets. @@ -56,19 +56,18 @@ python antarctica_today download-tb > downloads that raw data. +#### Creates data: + +* `.nc` files in `Tb/` directory + + ## 2. Generate all the daily melt binary files -Generate new data in `data/daily_melt_bin_files/`: ```bash PYTHONPATH=. python antarctica_today generate-daily-melt ``` -> [!IMPORTANT] -> Binaries provided in this repo's `data/daily_melt_bin_files/` dir with pre-2016 dates -> are already calibrated by Tom Mote and don't need to be generated. Remember from the -> note in the previous step: pre-generated data goes through to 2022-01-10. -
🛠️ _TODO_ I receive a large number of warnings like: @@ -82,30 +81,27 @@ Why?
-## 3. Generate the database +#### Creates data: -This step creates, primarily, four pickle files: +* `.bin` files in `data/daily_melt_bin_files/` directory for dates on or after + 2022-01-10 -* `daily_cumulative_melt_averages.pickle` -* `daily_melt_pixel_averages.pickle` -* `database/v3_1979-present_gap_filled.pickle` -* `database/v3_1979-present_raw.pickle` +> [!IMPORTANT] +> Binaries provided in this repo's `data/daily_melt_bin_files/` dir with pre-2016 dates +> are already calibrated by Tom Mote and don't need to be generated. Remember from the +> note in the previous step: pre-generated data goes through to 2022-01-10. -Additionally: -* `.csv` files will be created in `database/` directory -* `.tif` files will be created in `data/mean_climatology/` directory -* `.tif` files will be created in `data/annual_*_geotifs` directories +## 3. Generate the database +> [!NOTE] +> This command may take up to tens of minutes. ```bash PYTHONPATH=. python antarctica_today preprocess ``` -> [!NOTE] -> This command may take up to tens of minutes. -
🛠️ _TODO_ - [ ] Why is the next section called "Initializing"? Are there multiple pickle files? @@ -124,11 +120,21 @@ python antarctica_today preprocess
-### Database initialization (?) +#### Creates data: +* `database/v3_1979-present_gap_filled.pickle` +* `database/v3_1979-present_raw.pickle` +* `database/gap_fill_data/daily_cumulative_melt_averages.pickle` +* `database/gap_fill_data/daily_melt_pixel_averages.pickle` +* `.csv` files in `database/` directory +* `.tif` files in `data/mean_climatology/` directory +* `.tif` files in `data/annual_*_geotifs/` directories + + +### Database initialization (?)
🛠️ _TODO_ -Is this step necessary? It seems like new files aren't being created when this step is +Is this step necessary? New files aren't being created when this step is run.
@@ -174,7 +180,7 @@ python antarctica_today/update_data.py > [!NOTE] > This command may take up to tens of minutes. -This will go through the entire database and produce summary maps and plots for every year on record in the `plots/` directory. +This will go through the entire database and produce summary maps and plots for every year on record. ```bash PYTHONPATH=. @@ -188,6 +194,11 @@ python antarctica_today process +#### Creates data: + +* `.png` files in `plots/` subdirectories + + ## Running in Docker This repository includes a `compose.yml` configuration which enables running this code @@ -196,3 +207,7 @@ with Docker. For example, the `download-tb` command can be run as follows: ```bash docker compose run cli download-tb ``` + +> [!WARNING] +> By default, outputs will be written as `root`! You can override the user (TODO: how?) +> to match your desired production user. From 7e549c02b0c44336f2fa93170ff4b747f98ff48c Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 24 Jul 2024 13:06:12 -0600 Subject: [PATCH 3/3] Remove PyPI build GHActions jobs --- .github/workflows/build-and-publish.yml | 30 ------------------------- 1 file changed, 30 deletions(-) diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index dccba06..b783ac0 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -18,36 +18,6 @@ env: FORCE_COLOR: 3 jobs: - # build-python-package: - # name: Build Python package - # runs-on: ubuntu-latest - # steps: - # - uses: actions/checkout@v4 - # with: - # fetch-depth: 0 - # - uses: hynek/build-and-inspect-python-package@v2 - - # publish-python-package: - # name: Publish Python package to PyPI - # if: github.ref_type == 'tag' - # needs: ["build-python-package"] - # runs-on: ubuntu-latest - # environment: pypi - # permissions: - # id-token: write - # steps: - # - uses: actions/download-artifact@v4 - # with: - # name: Packages - # path: dist - - # - uses: pypa/gh-action-pypi-publish@release/v1 - # if: github.event_name == 'release' && github.event.action == 'published' - # with: - # # Remember to tell (test-)pypi about this repo before publishing - # # Remove this line to publish to PyPI - # repository-url: https://test.pypi.org/legacy/ - build-and-publish-docker-image: name: "Build and publish Docker image" uses: "nsidc/.github/.github/workflows/build-and-publish-container-image.yml@main"