From 83f5cfa1d7e82338eb9a47788d4999c4c16eda14 Mon Sep 17 00:00:00 2001 From: Rasool Saghaleyni Date: Fri, 13 Sep 2024 22:56:46 +0200 Subject: [PATCH] Add workflow for making docker images --- .../Build and Push Docker Images for Labs.yml | 115 ++++++++++++++++++ .../docker-build-lab-Single_cell-amd.yml | 107 +++++++--------- ...uild-lab-deep-learning-integration-amd.yml | 88 +++++++------- ...docker-build-lab-feature-selection-amd.yml | 44 +++++++ .../workflows/docker-build-lab-gems-amd.yml | 107 +++++++--------- .../FeatureSelectionIntegrOMICs/Dockerfile | 8 ++ session_ml/SingleCell/SingleCell_env.yml | 3 +- 7 files changed, 301 insertions(+), 171 deletions(-) create mode 100644 .github/workflows/Build and Push Docker Images for Labs.yml create mode 100644 .github/workflows/docker-build-lab-feature-selection-amd.yml diff --git a/.github/workflows/Build and Push Docker Images for Labs.yml b/.github/workflows/Build and Push Docker Images for Labs.yml new file mode 100644 index 0000000..7216790 --- /dev/null +++ b/.github/workflows/Build and Push Docker Images for Labs.yml @@ -0,0 +1,115 @@ +name: Build and Push Docker Images for Labs + +on: + push: + paths: + - 'session_ml/DeepLearningDataIntegration/**' + - 'session_ml/FeatureSelectionIntegrOMICs/**' + - 'session_ml/SingleCell/**' + - 'session_gems/**' + branches: + - OMICSINT_H24 + +jobs: + build: + runs-on: ubuntu-latest + + if: github.repository == 'NBISweden/workshop_omics_integration' + + steps: + # Checkout the repository + - name: Checkout repository + uses: actions/checkout@v3 + + # Set up Docker Buildx for multi-platform builds + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + # Log in to Docker Hub + - name: Log in to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + + # Log in to GitHub Container Registry (GHCR) + - name: Log in to GHCR + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + # Determine which directories have changes + - name: Check for Changes + id: changes + run: | + echo "## Detecting changes in the repository..." + if git diff --name-only HEAD^ HEAD | grep -q 'session_ml/DeepLearningDataIntegration'; then + echo "deep_learning=true" >> $GITHUB_ENV + fi + if git diff --name-only HEAD^ HEAD | grep -q 'session_ml/FeatureSelectionIntegrOMICs'; then + echo "feature_selection=true" >> $GITHUB_ENV + fi + if git diff --name-only HEAD^ HEAD | grep -q 'session_ml/SingleCell'; then + echo "single_cell=true" >> $GITHUB_ENV + fi + if git diff --name-only HEAD^ HEAD | grep -q 'session_gems'; then + echo "gems=true" >> $GITHUB_ENV + fi + + # Conditionally build and push images for Deep Learning Lab + - name: Build and Push Docker Image for Deep Learning Lab + if: env.deep_learning == 'true' + uses: docker/build-push-action@v4 + with: + context: ./session_ml/DeepLearningDataIntegration + file: session_ml/DeepLearningDataIntegration/Dockerfile + platforms: linux/amd64 + push: true + no-cache: true + tags: | + docker.io/rasoolsnbis/omicsint_h24:session_ml_deep_learning_integration_amd + ghcr.io/nbisweden/omicsint_h24:session_ml_deep_learning_integration_amd + + # Conditionally build and push images for Feature Selection Lab + - name: Build and Push Docker Image for Feature Selection Lab + if: env.feature_selection == 'true' + uses: docker/build-push-action@v4 + with: + context: ./session_ml/FeatureSelectionIntegrOMICs + file: session_ml/FeatureSelectionIntegrOMICs/Dockerfile + platforms: linux/amd64 + push: true + no-cache: true + tags: | + docker.io/rasoolsnbis/omicsint_h24:session_ml_feature_selection_amd + ghcr.io/nbisweden/omicsint_h24:session_ml_feature_selection_amd + + # Conditionally build and push images for Single Cell Lab + - name: Build and Push Docker Image for Single Cell Lab + if: env.single_cell == 'true' + uses: docker/build-push-action@v4 + with: + context: ./session_ml/SingleCell + file: session_ml/SingleCell/Dockerfile + platforms: linux/amd64 + push: true + no-cache: true + tags: | + docker.io/rasoolsnbis/omicsint_h24:session_ml_single_cell_amd + ghcr.io/nbisweden/omicsint_h24:session_ml_single_cell_amd + + # Conditionally build and push images for Gems Lab + - name: Build and Push Docker Image for Gems Lab + if: env.gems == 'true' + uses: docker/build-push-action@v4 + with: + context: ./session_gems + file: session_gems/Dockerfile + platforms: linux/amd64 + push: true + no-cache: true + tags: | + docker.io/rasoolsnbis/omicsint_h24:session_gems_amd + ghcr.io/nbisweden/omicsint_h24:session_gems_amd diff --git a/.github/workflows/docker-build-lab-Single_cell-amd.yml b/.github/workflows/docker-build-lab-Single_cell-amd.yml index c1bf842..e51794b 100644 --- a/.github/workflows/docker-build-lab-Single_cell-amd.yml +++ b/.github/workflows/docker-build-lab-Single_cell-amd.yml @@ -1,63 +1,44 @@ -name: Build and Push Docker Image for lab single cell - -on: - push: - paths: - - 'session_ml/SingleCell/**' - branches: - - OMICSINT_H24 - -jobs: - build: - runs-on: ubuntu-latest - - if: github.repository == 'nbisweden/workshop_omics_integration' - - steps: - # Checkout the repository - - name: Checkout repository - uses: actions/checkout@v3 - - # Set up Docker Buildx for multi-platform builds - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - # Log in to Docker Hub using the secrets - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - # Log in to GitHub Container Registry (GHCR) - - name: Log in to GHCR - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Build and push the Docker image for both amd64 to Docker Hub - - name: Build and Push Docker Image to Docker Hub - uses: docker/build-push-action@v4 - with: - context: ./session_ml/SingleCell - file: session_ml/SingleCell/Dockerfile - platforms: linux/amd64 - push: true - no-cache: true - tags: docker.io/rasoolsnbis/omicsint_h24:session_ml_single_cell_amd - - # Build and push the Docker image to GitHub Container Registry (GHCR) - - name: Build and Push Docker Image to GHCR - uses: docker/build-push-action@v4 - with: - context: ./session_ml/SingleCell - file: session_ml/SingleCell/Dockerfile - platforms: linux/amd64 - push: true - no-cache: true - tags: ghcr.io/nbisweden/omicsint_h24:session_ml_single_cell_amd - - - name: Inspect Docker Image - run: docker buildx imagetools inspect docker.io/rasoolsnbis/omicsint_h24:session_ml_single_cell_amd +# name: Build and Push Docker Image for lab single cell + +# on: +# push: +# paths: +# - 'session_ml/SingleCell/**' +# branches: +# - OMICSINT_H24 + +# jobs: +# build: +# runs-on: ubuntu-latest + +# if: github.repository == 'nbisweden/workshop_omics_integration' + +# steps: +# # Checkout the repository +# - name: Checkout repository +# uses: actions/checkout@v3 + +# # Set up Docker Buildx for multi-platform builds +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v2 + +# # Log in to Docker Hub using the secrets +# - name: Log in to Docker Hub +# uses: docker/login-action@v2 +# with: +# username: ${{ secrets.DOCKER_HUB_USERNAME }} +# password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + +# # Build and push the Docker image for both amd64 to Docker Hub +# - name: Build and Push Docker Image to Docker Hub +# uses: docker/build-push-action@v4 +# with: +# context: ./session_ml/SingleCell +# file: session_ml/SingleCell/Dockerfile +# platforms: linux/amd64 +# push: true +# no-cache: true +# tags: docker.io/rasoolsnbis/omicsint_h24:session_ml_single_cell_amd + +# - name: Inspect Docker Image +# run: docker buildx imagetools inspect docker.io/rasoolsnbis/omicsint_h24:session_ml_single_cell_amd diff --git a/.github/workflows/docker-build-lab-deep-learning-integration-amd.yml b/.github/workflows/docker-build-lab-deep-learning-integration-amd.yml index 251785d..331129a 100644 --- a/.github/workflows/docker-build-lab-deep-learning-integration-amd.yml +++ b/.github/workflows/docker-build-lab-deep-learning-integration-amd.yml @@ -1,44 +1,44 @@ -name: Build and Push Docker Image for lab deep learning integration - -on: - push: - paths: - - 'session_ml/DeepLearningDataIntegration/**' - branches: - - OMICSINT_H24 - -jobs: - build: - runs-on: ubuntu-latest - - if: github.repository == 'NBISweden/workshop_omics_integration' - - steps: - # Checkout the repository - - name: Checkout repository - uses: actions/checkout@v3 - - # Set up Docker Buildx for multi-platform builds - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - # Log in to Docker Hub using the secrets - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - # Build and push the Docker image for both amd64 and arm64 - - name: Build and Push Docker Image for Session Gems - uses: docker/build-push-action@v4 - with: - context: ./session_ml/DeepLearningDataIntegration - file: session_ml/DeepLearningDataIntegration/Dockerfile - platforms: linux/amd64 - push: true - no-cache: true - tags: docker.io/rasoolsnbis/omicsint_h24:session_ml_deep_learning_integration_amd - - - name: Inspect Docker Image - run: docker buildx imagetools inspect docker.io/rasoolsnbis/omicsint_h24:session_ml_deep_learning_integration_amd +# name: Build and Push Docker Image for lab deep learning integration + +# on: +# push: +# paths: +# - 'session_ml/DeepLearningDataIntegration/**' +# branches: +# - OMICSINT_H24 + +# jobs: +# build: +# runs-on: ubuntu-latest + +# if: github.repository == 'NBISweden/workshop_omics_integration' + +# steps: +# # Checkout the repository +# - name: Checkout repository +# uses: actions/checkout@v3 + +# # Set up Docker Buildx for multi-platform builds +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v2 + +# # Log in to Docker Hub using the secrets +# - name: Log in to Docker Hub +# uses: docker/login-action@v2 +# with: +# username: ${{ secrets.DOCKER_HUB_USERNAME }} +# password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + +# # Build and push the Docker image for both amd64 and arm64 +# - name: Build and Push Docker Image for Session Gems +# uses: docker/build-push-action@v4 +# with: +# context: ./session_ml/DeepLearningDataIntegration +# file: session_ml/DeepLearningDataIntegration/Dockerfile +# platforms: linux/amd64 +# push: true +# no-cache: true +# tags: docker.io/rasoolsnbis/omicsint_h24:session_ml_deep_learning_integration_amd + +# - name: Inspect Docker Image +# run: docker buildx imagetools inspect docker.io/rasoolsnbis/omicsint_h24:session_ml_deep_learning_integration_amd diff --git a/.github/workflows/docker-build-lab-feature-selection-amd.yml b/.github/workflows/docker-build-lab-feature-selection-amd.yml new file mode 100644 index 0000000..01cee54 --- /dev/null +++ b/.github/workflows/docker-build-lab-feature-selection-amd.yml @@ -0,0 +1,44 @@ +# name: Build and Push Docker Image for lab feature selection + +# on: +# push: +# paths: +# - 'session_ml/FeatureSelectionIntegrOMICs/**' +# branches: +# - OMICSINT_H24 + +# jobs: +# build: +# runs-on: ubuntu-latest + +# if: github.repository == 'nbisweden/workshop_omics_integration' + +# steps: +# # Checkout the repository +# - name: Checkout repository +# uses: actions/checkout@v3 + +# # Set up Docker Buildx for multi-platform builds +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v2 + +# # Log in to Docker Hub using the secrets +# - name: Log in to Docker Hub +# uses: docker/login-action@v2 +# with: +# username: ${{ secrets.DOCKER_HUB_USERNAME }} +# password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + +# # Build and push the Docker image for both amd64 to Docker Hub +# - name: Build and Push Docker Image to Docker Hub +# uses: docker/build-push-action@v4 +# with: +# context: ./session_ml/FeatureSelectionIntegrOMICs +# file: session_ml/FeatureSelectionIntegrOMICs/Dockerfile +# platforms: linux/amd64 +# push: true +# no-cache: true +# tags: docker.io/rasoolsnbis/omicsint_h24:session_ml_feature_selection_amd + +# - name: Inspect Docker Image +# run: docker buildx imagetools inspect docker.io/rasoolsnbis/omicsint_h24:session_ml_feature_selection_amd diff --git a/.github/workflows/docker-build-lab-gems-amd.yml b/.github/workflows/docker-build-lab-gems-amd.yml index 79aa510..0ae124f 100644 --- a/.github/workflows/docker-build-lab-gems-amd.yml +++ b/.github/workflows/docker-build-lab-gems-amd.yml @@ -1,63 +1,44 @@ -name: Build and Push Docker Image for Session Gems - -on: - push: - paths: - - 'session_gems/**' - branches: - - OMICSINT_H24 - -jobs: - build: - runs-on: ubuntu-latest - - if: github.repository == 'NBISweden/workshop_omics_integration' - - steps: - # Checkout the repository - - name: Checkout repository - uses: actions/checkout@v3 - - # Set up Docker Buildx for multi-platform builds - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 - - # Log in to Docker Hub using the secrets - - name: Log in to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ secrets.DOCKER_HUB_USERNAME }} - password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} - - # Log in to GitHub Container Registry (GHCR) - - name: Log in to GHCR - uses: docker/login-action@v2 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - # Build and push the Docker image for both amd64 and arm64 - - name: Build and Push Docker Image for Session Gems - uses: docker/build-push-action@v4 - with: - context: ./session_gems - file: session_gems/Dockerfile - platforms: linux/amd64 - push: true - no-cache: true - tags: docker.io/rasoolsnbis/omicsint_h24:session_gems_amd - - # Build and push the Docker image to GitHub Container Registry (GHCR) - - name: Build and Push Docker Image to GHCR - uses: docker/build-push-action@v4 - with: - context: ./session_gems - file: session_gems/Dockerfile - platforms: linux/amd64 - push: true - no-cache: true - tags: ghcr.io/nbisweden/omicsint_h24:session_gems_amd - - - name: Inspect Docker Image - run: docker buildx imagetools inspect docker.io/rasoolsnbis/omicsint_h24:session_gems_amd +# name: Build and Push Docker Image for Session Gems + +# on: +# push: +# paths: +# - 'session_gems/**' +# branches: +# - OMICSINT_H24 + +# jobs: +# build: +# runs-on: ubuntu-latest + +# if: github.repository == 'NBISweden/workshop_omics_integration' + +# steps: +# # Checkout the repository +# - name: Checkout repository +# uses: actions/checkout@v3 + +# # Set up Docker Buildx for multi-platform builds +# - name: Set up Docker Buildx +# uses: docker/setup-buildx-action@v2 + +# # Log in to Docker Hub using the secrets +# - name: Log in to Docker Hub +# uses: docker/login-action@v2 +# with: +# username: ${{ secrets.DOCKER_HUB_USERNAME }} +# password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }} + +# # Build and push the Docker image for both amd64 and arm64 +# - name: Build and Push Docker Image for Session Gems +# uses: docker/build-push-action@v4 +# with: +# context: ./session_gems +# file: session_gems/Dockerfile +# platforms: linux/amd64 +# push: true +# no-cache: true +# tags: docker.io/rasoolsnbis/omicsint_h24:session_gems_amd + +# - name: Inspect Docker Image +# run: docker buildx imagetools inspect docker.io/rasoolsnbis/omicsint_h24:session_gems_amd diff --git a/session_ml/FeatureSelectionIntegrOMICs/Dockerfile b/session_ml/FeatureSelectionIntegrOMICs/Dockerfile index 9e45d6e..fc924a8 100644 --- a/session_ml/FeatureSelectionIntegrOMICs/Dockerfile +++ b/session_ml/FeatureSelectionIntegrOMICs/Dockerfile @@ -26,6 +26,7 @@ RUN export DEBIAN_FRONTEND=noninteractive \ libxml2 \ libodbc1 \ libglpk-dev \ + wget \ && echo "en_US.UTF-8 UTF-8" > /etc/locale.gen \ && locale-gen \ && rm -rf /var/lib/apt/lists/* @@ -66,6 +67,13 @@ COPY FeatureSelection_env.yml /tmp/FeatureSelection_env.yml RUN mamba env create -f /tmp/FeatureSelection_env.yml \ && rm /tmp/FeatureSelection_env.yml +# Download required files to the data directory +RUN mkdir -p /home/jovyan/lab/data \ + && wget -O /home/jovyan/lab/data/GTEX_SkeletalMuscles_157Samples_1000Genes.txt "https://github.com/NBISweden/workshop_omics_integration/blob/main/session_ml/FeatureSelectionIntegrOMICs/GTEX_SkeletalMuscles_157Samples_1000Genes.txt" \ + && wget -O /home/jovyan/lab/data/GTEX_SkeletalMuscles_157Samples_Gender.txt "https://github.com/NBISweden/workshop_omics_integration/blob/main/session_ml/FeatureSelectionIntegrOMICs/GTEX_SkeletalMuscles_157Samples_Gender.txt" \ + && wget -O /home/jovyan/lab/data/aGTEx_Data_2014-01-17_Annotations_SubjectPhenotypes_DS.txt "https://github.com/NBISweden/workshop_omics_integration/blob/main/session_ml/FeatureSelectionIntegrOMICs/GTEx_Data_2014-01-17_Annotations_SubjectPhenotypes_DS.txt" \ + && wget -O /home/jovyan/lab/data/TMM_NormalizedCounts_157_Samples_Muscle.txt "https://github.com/NBISweden/workshop_omics_integration/blob/main/session_ml/FeatureSelectionIntegrOMICs/TMM_NormalizedCounts_157_Samples_Muscle.txt" + # Download and install RStudio Server RUN curl -sL "https://download2.rstudio.org/server/focal/${RSTUDIO_ARCH}/rstudio-server-${RSTUDIO_VERSION//+/-}-${RSTUDIO_ARCH}.deb" -o /tmp/rstudio-server.deb && \ echo "64044984a5791690586e21bf4411019ed3dccaf32a69e0838090762f3eba8da0 /tmp/rstudio-server.deb" | sha256sum -c - && \ diff --git a/session_ml/SingleCell/SingleCell_env.yml b/session_ml/SingleCell/SingleCell_env.yml index 31646ef..43920ff 100644 --- a/session_ml/SingleCell/SingleCell_env.yml +++ b/session_ml/SingleCell/SingleCell_env.yml @@ -14,4 +14,5 @@ dependencies: - r-seurat - r-signac - r-cowplot - - r-hdf5r \ No newline at end of file + - r-hdf5r + \ No newline at end of file