From 7fcc6a62b1762fda0494285d42f39d7ddcb88e5d Mon Sep 17 00:00:00 2001 From: Carlos Paniagua Date: Sat, 17 Feb 2024 02:19:10 -0500 Subject: [PATCH 1/7] Update Dockerfile to use Python 3.11 image and install required packages --- Dockerfile | 60 ++++++++++++++++-------------------------------------- 1 file changed, 17 insertions(+), 43 deletions(-) diff --git a/Dockerfile b/Dockerfile index dd0de0d7..466445eb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,48 +1,22 @@ -# Using the ubuntu:20.04 base image for compatibility with the ICESat-2 SlideRule software -# Some code to add conda from https://github.com/conda-forge/miniforge-images/blob/master/ubuntu/Dockerfile -# This file builds the docker image for the ICESat-2 tracks project. Includes the ICESat-2 SlideRule tool, conda, and the python3.11 environment for the project with dependencies provided in the environment_small.yml file. -FROM ubuntu:20.04 -ARG DEBIAN_FRONTEND=noninteractive -ARG MINIFORGE_NAME=Miniforge3 -ARG MINIFORGE_VERSION=23.3.1-0 -ARG TARGETPLATFORM +# Use the official Python 3.11 image from the Docker Hub +FROM python:3.11 -ENV CONDA_DIR=/opt/conda -ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 -ENV PATH=${CONDA_DIR}/bin:${PATH} -ENV ENVNAME=2021-icesat2-tracks -RUN apt-get update && \ - apt-get install -y git curl wget +# Update the system and install the packages +RUN apt-get update && apt-get install -y \ + openmpi-bin \ + openmpi-doc \ + libopenmpi-dev -# Install sliderule dependencies -RUN apt-get install -y build-essential libreadline-dev liblua5.3-dev zlib1g-dev cmake -RUN git clone https://github.com/ICESat2-SlideRule/sliderule.git -WORKDIR /sliderule/ -RUN make config && \ - make && \ - make install +# Set the working directory to /app +WORKDIR /app -# Install conda -RUN apt-get update > /dev/null && \ - apt-get install --no-install-recommends --yes \ - bzip2 ca-certificates \ - tini \ - > /dev/null && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* && \ - wget --no-hsts --quiet https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/${MINIFORGE_NAME}-${MINIFORGE_VERSION}-Linux-$(uname -m).sh -O /tmp/miniforge.sh && \ - /bin/bash /tmp/miniforge.sh -b -p ${CONDA_DIR} && \ - rm /tmp/miniforge.sh && \ - conda clean --tarballs --index-cache --packages --yes && \ - find ${CONDA_DIR} -follow -type f -name '*.a' -delete && \ - find ${CONDA_DIR} -follow -type f -name '*.pyc' -delete && \ - conda clean --force-pkgs-dirs --all --yes && \ - echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \ - echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc +# Copy the current directory contents into the container at /app +COPY . /app -# Install conda environment -COPY environment_small.yml /tmp/environment.yml -RUN mamba env create -f /tmp/environment.yml +# Create the output directory +RUN mkdir -p /app/output -ENTRYPOINT ["tini", "--"] -CMD ["/bin/bash"] +# Install any needed packages specified in pyproject.toml +RUN pip install . + +CMD ["/bin/bash"] \ No newline at end of file From 99fdc59be80320439137a06f9b76d7ead99c108d Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 4 Mar 2024 22:12:06 +0000 Subject: [PATCH 2/7] add Dockerfile and devcontainer.json arguments to make install work --- .devcontainer/devcontainer.json | 26 ++++++++++++++++++++++++++ Dockerfile | 10 +++++++--- 2 files changed, 33 insertions(+), 3 deletions(-) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..8d96444e --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,26 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the +// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile +{ + "name": "Existing Dockerfile", + "build": { + // Sets the run context to one level up instead of the .devcontainer folder. + "context": "..", + // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. + "dockerfile": "../Dockerfile" + } + + // Features to add to the dev container. More info: https://containers.dev/features. + // "features": {}, + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Uncomment the next line to run commands after the container is created. + // "postCreateCommand": "cat /etc/os-release", + + // Configure tool-specific properties. + // "customizations": {}, + + // Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root. + // "remoteUser": "devcontainer" +} diff --git a/Dockerfile b/Dockerfile index 466445eb..ef57b371 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,11 +1,15 @@ # Use the official Python 3.11 image from the Docker Hub -FROM python:3.11 +FROM python:3.9 # Update the system and install the packages RUN apt-get update && apt-get install -y \ + git-lfs \ openmpi-bin \ openmpi-doc \ - libopenmpi-dev + libopenmpi-dev \ + gdal-bin \ + libgdal-dev + # Set the working directory to /app WORKDIR /app @@ -17,6 +21,6 @@ COPY . /app RUN mkdir -p /app/output # Install any needed packages specified in pyproject.toml -RUN pip install . +RUN pip install --editable ".[dev]" CMD ["/bin/bash"] \ No newline at end of file From 268377c698c5e5f1662991bc3de9032e9eaa4063 Mon Sep 17 00:00:00 2001 From: John Gerrard Holland Date: Mon, 4 Mar 2024 22:44:45 +0000 Subject: [PATCH 3/7] =?UTF-8?q?add=20GDAL=20and=20LFS=20=E2=80=93=20make?= =?UTF-8?q?=20docker=20work?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Dockerfile => .devcontainer/Dockerfile | 2 +- .devcontainer/devcontainer.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) rename Dockerfile => .devcontainer/Dockerfile (96%) diff --git a/Dockerfile b/.devcontainer/Dockerfile similarity index 96% rename from Dockerfile rename to .devcontainer/Dockerfile index ef57b371..ccff69f5 100644 --- a/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,5 +1,5 @@ # Use the official Python 3.11 image from the Docker Hub -FROM python:3.9 +FROM python:3.11 # Update the system and install the packages RUN apt-get update && apt-get install -y \ diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8d96444e..43a4a24b 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,7 +6,7 @@ // Sets the run context to one level up instead of the .devcontainer folder. "context": "..", // Update the 'dockerFile' property if you aren't using the standard 'Dockerfile' filename. - "dockerfile": "../Dockerfile" + "dockerfile": "Dockerfile" } // Features to add to the dev container. More info: https://containers.dev/features. From bb7461522d87bc90da99b6be7568b3737ab29d16 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 20:32:54 +0000 Subject: [PATCH 4/7] chore(deps): bump actions/checkout from 2 to 4 Bumps [actions/checkout](https://github.com/actions/checkout) from 2 to 4. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v2...v4) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/publish_pypi.yml | 2 +- .github/workflows/python_ci.yml | 2 +- .github/workflows/test_icesat2_tracks_pipeline.yml | 2 +- .github/workflows/test_test_suite.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 36399a28..f57b13bf 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -13,7 +13,7 @@ jobs: id-token: write steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v2 diff --git a/.github/workflows/python_ci.yml b/.github/workflows/python_ci.yml index 8bc33900..1459c4c6 100644 --- a/.github/workflows/python_ci.yml +++ b/.github/workflows/python_ci.yml @@ -15,7 +15,7 @@ jobs: - name: install mpi macos if: matrix.os == 'macos-latest' run: brew install open-mpi - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: setup python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/test_icesat2_tracks_pipeline.yml b/.github/workflows/test_icesat2_tracks_pipeline.yml index 736a96c2..1a2c883a 100644 --- a/.github/workflows/test_icesat2_tracks_pipeline.yml +++ b/.github/workflows/test_icesat2_tracks_pipeline.yml @@ -13,7 +13,7 @@ jobs: steps: - name: install mpi run: sudo apt update && sudo apt-get install openmpi-bin openmpi-doc libopenmpi-dev - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: setup python uses: actions/setup-python@v4 with: diff --git a/.github/workflows/test_test_suite.yml b/.github/workflows/test_test_suite.yml index 3ab9c0f3..ba19f179 100644 --- a/.github/workflows/test_test_suite.yml +++ b/.github/workflows/test_test_suite.yml @@ -13,7 +13,7 @@ jobs: steps: - name: install mpi run: sudo apt update && sudo apt-get install openmpi-bin openmpi-doc libopenmpi-dev - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: lfs: true - name: setup python From 8bbbb499e11c2d69befa06f4def1bd6e80325d26 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Mar 2024 21:43:37 +0000 Subject: [PATCH 5/7] chore(deps): bump actions/setup-python from 2 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 2 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v2...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/publish_pypi.yml | 2 +- .github/workflows/python_ci.yml | 2 +- .github/workflows/test_icesat2_tracks_pipeline.yml | 2 +- .github/workflows/test_test_suite.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish_pypi.yml b/.github/workflows/publish_pypi.yml index 4bd1f7da..56d8facf 100644 --- a/.github/workflows/publish_pypi.yml +++ b/.github/workflows/publish_pypi.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.11 diff --git a/.github/workflows/python_ci.yml b/.github/workflows/python_ci.yml index 1459c4c6..8923bff4 100644 --- a/.github/workflows/python_ci.yml +++ b/.github/workflows/python_ci.yml @@ -17,7 +17,7 @@ jobs: run: brew install open-mpi - uses: actions/checkout@v4 - name: setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.version }} # install the python version needed cache: "pip" diff --git a/.github/workflows/test_icesat2_tracks_pipeline.yml b/.github/workflows/test_icesat2_tracks_pipeline.yml index 1a2c883a..150f2bbe 100644 --- a/.github/workflows/test_icesat2_tracks_pipeline.yml +++ b/.github/workflows/test_icesat2_tracks_pipeline.yml @@ -15,7 +15,7 @@ jobs: run: sudo apt update && sudo apt-get install openmpi-bin openmpi-doc libopenmpi-dev - uses: actions/checkout@v4 - name: setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.version }} # install the python version needed cache: "pip" diff --git a/.github/workflows/test_test_suite.yml b/.github/workflows/test_test_suite.yml index ba19f179..674211a6 100644 --- a/.github/workflows/test_test_suite.yml +++ b/.github/workflows/test_test_suite.yml @@ -17,7 +17,7 @@ jobs: with: lfs: true - name: setup python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.version }} # install the python version needed cache: "pip" From 5ecfe893267436198c8c86cca1afeaa9cddcdbad Mon Sep 17 00:00:00 2001 From: Camilo Diaz Date: Tue, 5 Mar 2024 16:50:41 -0500 Subject: [PATCH 6/7] added table of contents in readme --- README.md | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/README.md b/README.md index c0a2c488..2590eee5 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,16 @@ # ICESAT2 Track Analysis +- [ICESAT2 Track Analysis](#icesat2-track-analysis) + - [Installation for Developers](#installation-for-developers) + - [Installing on Oscar (Deprecated)](#installing-on-oscar-deprecated) + - [Conda Configuration (Deprecated)](#conda-configuration-deprecated) + - [`.condarc`](#condarc) + - [`pkgs_dirs`](#pkgs_dirs) + - [`envs_dirs`](#envs_dirs) + - [Environment Variables](#environment-variables) + - [Command line interface](#command-line-interface) + - [Sample workflow](#sample-workflow) + ## Installation for Developers Prerequisites: From 7fad6550574e3fb11a224e523b46109ed41afcb1 Mon Sep 17 00:00:00 2001 From: Camilo Diaz Date: Tue, 5 Mar 2024 18:33:50 -0500 Subject: [PATCH 7/7] removed deprecated sections from readme --- README.md | 157 ------------------------------------------------------ 1 file changed, 157 deletions(-) diff --git a/README.md b/README.md index 2590eee5..083700d9 100644 --- a/README.md +++ b/README.md @@ -2,12 +2,6 @@ - [ICESAT2 Track Analysis](#icesat2-track-analysis) - [Installation for Developers](#installation-for-developers) - - [Installing on Oscar (Deprecated)](#installing-on-oscar-deprecated) - - [Conda Configuration (Deprecated)](#conda-configuration-deprecated) - - [`.condarc`](#condarc) - - [`pkgs_dirs`](#pkgs_dirs) - - [`envs_dirs`](#envs_dirs) - - [Environment Variables](#environment-variables) - [Command line interface](#command-line-interface) - [Sample workflow](#sample-workflow) @@ -66,157 +60,6 @@ Installation: python -c "import icesat2_tracks; print(icesat2_tracks.__version__)" ``` -## Installing on Oscar (Deprecated) - -If any of these commands fail, check the conda configuration (listed below) before retrying for possible fixes. - -Load a conda module. - -```shell -module load miniconda/23.1.0 -``` - -Follow any prompts from the module load command, like running the following line: -```shell -source /gpfs/runtime/opt/miniconda/23.1.0/etc/profile.d/conda.sh -``` - -Create a new environment using: -```shell -conda create --name "2021-icesat2-tracks" -``` - -Activate the environment using: -```shell -conda activate "2021-icesat2-tracks" -``` - -Install or update the packages in the environment with those listed in the `environment.yml` file using: -```shell -conda env update --file environment.yml -``` - -(You can create and install dependencies in the environment in a single command using: -```shell -conda env create --name "2021-icesat2-tracks" --file environment.yml -``` -... but this has more steps and is thus more likely to fail. Since the installation step takes a long period of time, it is recommended to use the separate commands instead.) - -## Conda Configuration (Deprecated) - -Conda draws its configuration from multiple places, and will behave differently when the configuration is different, even when using the same `environment.yml` file. - -#### `.condarc` - -The `.condarc` file in your home directory sets your conda configuration. If the file doesn't exist, you can create it with: -```shell -touch ~/.condarc -``` - -#### `pkgs_dirs` - -`pkgs_dirs` is the location where conda downloads package files from registries like `anaconda.org`. - -If you use the defaults, when trying to install packages you may get a warning like: -``` -WARNING conda.lock:touch(51): Failed to create lock, do not run conda in parallel processes [errno 13] -... -ERROR Could not open file /gpfs/runtime/opt/miniconda/4.12.0/pkgs/cache/b63425f9.json -``` - -In this case, you might be trying to download packages to the global directory `/gpfs/runtime/opt/miniconda/4.12.0/pkgs` where you have no write-permissions, rather than your home directory where you have write-permissions. - -View the conda configuration: -```shell -conda config --show -``` - -Check that the `pkgs_dirs` setting points to a location in your home directory: -```yaml -pkgs_dirs: - - /users/username/anaconda/pkg -``` - -If it doesn't, update this using: -```shell -conda config --add pkgs_dirs ~/anaconda/pkg -``` - -(Use `--remove` instead of `--add` to remove an entry.) - -#### `envs_dirs` - -`envs_dirs` is the location where there is a separate directory per environment containing the installed packages. - -View the conda configuration: -```shell -conda config --show -``` - -Check that the `envs_dirs` setting to a location in your home directory: -```yaml -envs_dirs: - - /users/username/anaconda/envs -``` - -... and update this using: -```shell -conda config --add envs_dirs ~/anaconda/envs -``` - -(Use `--remove` instead of `--add` to remove an entry.) - -Always re-check the configuration after running the `conda config` command. - -#### Environment Variables - -Note that modules (like `miniconda/23.1.0`) set environment variables like `CONDA_ENVS_PATH` which override the conda config. - -You might view the conda config and see the following entries: -```yaml -envs_dirs: - - /users/username/anaconda - - /users/username/anaconda/envs -``` - -If you try to run -```shell -conda config --remove envs_dirs ~/anaconda -``` -... you'll get a warning: -``` -'envs_dirs': '/users/username/anaconda' is not in the 'envs_dirs' key of the config file -``` - -... and find that the value is still there when you rerun `conda config --show`: -```yaml -envs_dirs: - - /users/username/anaconda # <-- still here! - - /users/username/anaconda/envs -``` - -The value might have been silently set by the `module load` command using an environment variable. - -Check for environment variables by running: -```shell -$ printenv | grep ^CONDA_ -CONDA_SHLVL=0 -CONDA_EXE=/gpfs/runtime/opt/miniconda/23.1.0/bin/conda -CONDA_ENVS_PATH=~/anaconda # <- this is the offending variable -CONDA_PYTHON_EXE=/gpfs/runtime/opt/miniconda/23.1.0/bin/python -``` - -To unset a value like `CONDA_ENVS_PATH` use: -```shell -unset CONDA_ENVS_PATH -``` - -... then check that rerun `conda config --show` no longer shows the has modified the conda config to match the values you wanted: -```yaml -envs_dirs: - - /users/username/anaconda/envs -``` - ## Command line interface The `icesat2waves` package comes with a command-line interface (CLI) that facilitates interaction with the package directly from your terminal. This can be particularly useful for scripting and automation. You can access the help documentation for the CLI by running the following command: