From f2034e4d0e3b5595b335bdc9d377cf461085cdcf Mon Sep 17 00:00:00 2001 From: Ali Khan Date: Thu, 21 Sep 2023 11:46:08 -0400 Subject: [PATCH] Pythondeps (#8) * include all hippunfold pythondeps, update dockerfile, replace circleci with gh actions so we can remove these dependencies from the hippunfold repo note: still WIP --- .circleci/config.yml | 225 ---------------------------- .github/workflows/deploy_docker.yml | 52 +++++++ Dockerfile | 8 + autotop_deps/__init__.py | 0 pyproject.toml | 28 ++++ 5 files changed, 88 insertions(+), 225 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/deploy_docker.yml create mode 100644 autotop_deps/__init__.py create mode 100644 pyproject.toml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index d218f4f..0000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,225 +0,0 @@ -version: 2.1 -jobs: - build: - docker: - - image: docker:17.05.0-ce-git - steps: - - checkout - - setup_remote_docker - - run: - name: Building Docker container - no_output_timeout: 1h - command: | - if [ "$CIRCLE_BRANCH" = "master" -o "$CIRCLE_BRANCH" = "" ]; then MY_TAG=latest; else MY_TAG=$CIRCLE_BRANCH; fi - export DOCKER_NAME=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME:$MY_TAG - docker build -t $DOCKER_NAME . - docker save $DOCKER_NAME -o /tmp/docker_image.tar - - persist_to_workspace: - root: /tmp - paths: - - docker_image.tar - - build_sif: - docker: - - image: singularityware/singularity:3.2.1-slim - steps: - - checkout - - attach_workspace: - at: /tmp/workspace - - run: - name: Building singularity image from docker tar - no_output_timeout: 1h - command: | - singularity build /tmp/singularity_image.sif docker-archive:///tmp/workspace/docker_image.tar - - persist_to_workspace: - root: /tmp - paths: - - singularity_image.sif - - test: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker - - attach_workspace: - at: /tmp/workspace - - run: - name: Install dependencies - command: | - apk add --update make wget - - run: - name: Load docker image - no_output_timeout: 1h - command: | - docker load -i /tmp/workspace/docker_image.tar - - run: - name: Running Tests - command: | - if [ "$CIRCLE_BRANCH" = "master" -o "$CIRCLE_BRANCH" = "" ]; then MY_TAG=latest; else MY_TAG=$CIRCLE_BRANCH; fi - export DOCKER_NAME=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME:$MY_TAG - # put tests here - echo "no tests implemented yet" - #docker run -it $DOCKER_NAME ls - test_sif: - docker: - - image: singularityware/singularity:3.2.1-slim - steps: - - attach_workspace: - at: /tmp/workspace - - run: - name: Install dependencies - command: | - apk add --update make wget - - run: - name: Running Tests - command: | - # put tests here -- note singularity exec doesn't seem to work right now.. - #singularity exec /tmp/workspace/singularity_image.sif ls - echo "no tests implemented yet" - - deploy_dev_cc: - docker: - - image: docker:17.05.0-ce-git - steps: - - add_ssh_keys: - fingerprints: - - "5a:4b:ef:15:70:14:a6:a1:0d:ca:4c:b8:e0:dd:a4:7d" - - attach_workspace: - at: /tmp/workspace - - run: - name: Push singularity image to compute canada - no_output_timeout: 1h - command: | - ssh-keyscan $CC_HOST >> ~/.ssh/known_hosts - if [ "$CIRCLE_BRANCH" = "master" ]; then MY_TAG=latest; else MY_TAG=$CIRCLE_BRANCH; fi - scp /tmp/workspace/singularity_image.sif ${CC_USER}@${CC_HOST}:${CC_PATH}/${CIRCLE_PROJECT_USERNAME}_${CIRCLE_PROJECT_REPONAME}_${MY_TAG}.sif - - deploy_dev_sif: - docker: - - image: singularityware/singularity:3.2.1-slim - steps: - - attach_workspace: - at: /tmp/workspace - - run: - name: Login to sylabs cloud - no_output_timeout: 1h - command: | - mkdir /root/.singularity - echo $SYLABS_TOKEN > /root/.singularity/sylabs-token - singularity remote list - - run: - name: Pushing build to sylabs cloud - command: | - if [ "$CIRCLE_BRANCH" = "master" ]; then MY_TAG=latest; else MY_TAG=$CIRCLE_BRANCH; fi - singularity push --allow-unsigned /tmp/workspace/singularity_image.sif library://${SYLABS_USER}/circleci/${CIRCLE_PROJECT_REPONAME}:$MY_TAG - - deploy_dev: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker - - attach_workspace: - at: /tmp/workspace - - run: - name: Load docker image - no_output_timeout: 1h - command: | - docker load -i /tmp/workspace/docker_image.tar - - run: - name: Pushing build to docker hub - command: | - docker login -u $DOCKER_USER -p $DOCKER_PASS - if [ "$CIRCLE_BRANCH" = "master" ]; then MY_TAG=latest; else MY_TAG=$CIRCLE_BRANCH; fi - export DOCKER_NAME=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME:$MY_TAG - # tag and push here: - echo docker push $DOCKER_NAME - docker push $DOCKER_NAME - - deploy_release: - docker: - - image: docker:17.05.0-ce-git - steps: - - setup_remote_docker - - attach_workspace: - at: /tmp/workspace - - run: - name: Load docker image - no_output_timeout: 1h - command: | - docker load -i /tmp/workspace/docker_image.tar - - run: - name: Pushing build to docker hub - command: | - docker login -u $DOCKER_USER -p $DOCKER_PASS - export DOCKER_NAME=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME:latest - export DOCKER_RELEASE=$CIRCLE_PROJECT_USERNAME/$CIRCLE_PROJECT_REPONAME:$CIRCLE_TAG - # tag and push here: - echo docker tag $DOCKER_NAME $DOCKER_RELEASE - docker tag $DOCKER_NAME $DOCKER_RELEASE - docker push $DOCKER_RELEASE - docker push $DOCKER_NAME - - deploy_release_cc: - docker: - - image: docker:17.05.0-ce-git - steps: - - add_ssh_keys: - fingerprints: - - "5a:4b:ef:15:70:14:a6:a1:0d:ca:4c:b8:e0:dd:a4:7d" - - attach_workspace: - at: /tmp/workspace - - run: - name: Push singularity image to compute canada - no_output_timeout: 1h - command: | - ssh-keyscan $CC_HOST >> ~/.ssh/known_hosts - scp /tmp/workspace/singularity_image.sif ${CC_USER}@${CC_HOST}:${CC_PATH}/${CIRCLE_PROJECT_USERNAME}_${CIRCLE_PROJECT_REPONAME}_${CIRCLE_TAG}.sif - - deploy_release_sif: - docker: - - image: singularityware/singularity:3.2.1-slim - steps: - - attach_workspace: - at: /tmp/workspace - - run: - name: Login to sylabs cloud - no_output_timeout: 1h - command: | - mkdir /root/.singularity - echo $SYLABS_TOKEN > /root/.singularity/sylabs-token - singularity remote list - - run: - name: Pushing release to sylabs cloud - command: | - singularity push --allow-unsigned /tmp/workspace/singularity_image.sif library://${SYLABS_USER}/circleci/${CIRCLE_PROJECT_REPONAME}:$CIRCLE_TAG - - -workflows: - version: 2.0 - commit: - jobs: - - build: - context: org-global - filters: - branches: - only: - - master - tags: - only: /.*/ - - deploy_dev: - requires: - - build - context: org-global - filters: - tags: - ignore: /^v.*/ - - deploy_release: - requires: - - build - context: org-global - filters: - tags: - only: /^v.*/ - branches: - ignore: /.*/ - diff --git a/.github/workflows/deploy_docker.yml b/.github/workflows/deploy_docker.yml new file mode 100644 index 0000000..aa60e8e --- /dev/null +++ b/.github/workflows/deploy_docker.yml @@ -0,0 +1,52 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +name: Publish Docker image + +on: + workflow_dispatch: + release: + types: [published] + +jobs: + push_to_registries: + name: Push Docker image to multiple registries + runs-on: ubuntu-latest + permissions: + packages: write + contents: read + steps: + - name: Check out the repo + uses: actions/checkout@v2 + + - name: Log in to Docker Hub + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Log in to the Container registry + uses: docker/login-action@f054a8b539a109f9f41c372932f1ae047eff08c9 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38 + with: + images: | + ${{ github.repository }} + ghcr.io/${{ github.repository }} + + - name: Build and push Docker images + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: . + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + diff --git a/Dockerfile b/Dockerfile index dc12758..a718f16 100644 --- a/Dockerfile +++ b/Dockerfile @@ -47,6 +47,14 @@ RUN cd /tmp/JCC-3.10 && python setup.py install && \ cd /opt && git clone http://github.com/nighres/nighres && cd /opt/nighres && git checkout 1901ce9a9afdfad8e2d66ec09600fbfb9fa0151d && ./build.sh && cd /opt/nighres && pip install . && \ rm -rf /opt/nighres /tmp/JCC-3.10 /tmp/JCC-3.10.tar.gz +COPY . /src + +ENV SKLEARN_ALLOW_DEPRECATED_SKLEARN_PACKAGE_INSTALL True +RUN pip install --no-cache-dir /src && \ + apt install -y graphviz && \ + wget https://imagemagick.org/archive/binaries/magick && \ + mv magick /usr/bin && chmod a+x /usr/bin/magick + ENV LD_LIBRARY_PATH /opt/itksnap/lib/:/opt/niftyreg-1.3.9/lib:/opt/workbench/libs_linux64:/opt/workbench/libs_linux64_software_opengl:${LD_LIBRARY_PATH} ENV PATH /opt/conda/bin:/opt/itksnap/bin/:/opt/niftyreg-1.3.9/bin:/opt/workbench/bin_linux64:/opt/ants-2.3.1-minify:/opt/fsl-5.0.11/bin-minify:$PATH diff --git a/autotop_deps/__init__.py b/autotop_deps/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..182158d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,28 @@ +[tool.poetry] +name = "autotop_deps" +version = "0.5.0" +description = "Dependencies for HippUnfold" +authors = ["Jordan DeKraker & Ali Khan "] + +[tool.poetry.dependencies] +python = ">=3.7.1,<3.10" +batchgenerators = "0.21" +nnunet-inference-on-cpu-and-gpu = "1.6.6" +numpy = "1.21.2" +astropy = "^4.0,<=4.3.1" +pandas = "^1.2.0,<=1.3.0" +nibabel = "^3.2.1" +scipy = "1.7.1" +nilearn = "^0.8.1" +seaborn = "^0.11.2" +scikit-fmm = "^2022.03.26" +simpleitk = "2.0.2" +matplotlib = "3.4.2" +torch = "1.10.0" + + +[tool.poetry.dev-dependencies] + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api"