From 310f76aa91769f69fbd296a8aa72ee9d82169969 Mon Sep 17 00:00:00 2001 From: Oscar Esteban Date: Tue, 9 Feb 2021 01:10:16 -0800 Subject: [PATCH] maint: refactor Dockerfile and move tests from travis to GHA --- .circleci/config.yml | 48 +++++++------- .github/workflows/pythonpackage.yml | 98 +++++++++++++++++++++++++++++ .travis.yml | 61 ------------------ Dockerfile | 70 +++++++++++---------- Makefile | 18 ++++++ 5 files changed, 176 insertions(+), 119 deletions(-) create mode 100644 .github/workflows/pythonpackage.yml delete mode 100644 .travis.yml create mode 100644 Makefile diff --git a/.circleci/config.yml b/.circleci/config.yml index be285477ad..e3215b6a64 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,10 +44,10 @@ jobs: steps: - restore_cache: keys: - - build-v2-{{ .Branch }}-{{ epoch }} - - build-v2-{{ .Branch }}- - - build-v2-master- - - build-v2- + - build-v3-{{ .Branch }}-{{ epoch }} + - build-v3-{{ .Branch }}- + - build-v3-master- + - build-v3- paths: - /tmp/docker - /tmp/images @@ -80,14 +80,14 @@ jobs: set -e if [[ "$success" = "0" ]]; then echo "Pulling from local registry" - docker tag localhost:5000/ubuntu ubuntu:xenial-20161213 + docker tag localhost:5000/ubuntu ubuntu:xenial-20201030 docker pull localhost:5000/smriprep docker tag localhost:5000/smriprep nipreps/smriprep:latest docker tag localhost:5000/smriprep nipreps/smriprep else echo "Pulling from Docker Hub" - docker pull ubuntu:xenial-20161213 - docker tag ubuntu:xenial-20161213 localhost:5000/ubuntu + docker pull ubuntu:xenial-20201030 + docker tag ubuntu:xenial-20201030 localhost:5000/ubuntu docker push localhost:5000/ubuntu docker pull nipreps/smriprep:latest fi @@ -130,7 +130,7 @@ jobs: docker exec -it registry /bin/registry garbage-collect --delete-untagged \ /etc/docker/registry/config.yml - save_cache: - key: build-v2-{{ .Branch }}-{{ epoch }} + key: build-v3-{{ .Branch }}-{{ epoch }} paths: - /tmp/docker - /tmp/images @@ -241,10 +241,10 @@ jobs: at: /tmp - restore_cache: keys: - - build-v2-{{ .Branch }}-{{ epoch }} - - build-v2-{{ .Branch }}- - - build-v2-master- - - build-v2- + - build-v3-{{ .Branch }}-{{ epoch }} + - build-v3-{{ .Branch }}- + - build-v3-master- + - build-v3- paths: - /tmp/docker - /tmp/images @@ -390,10 +390,10 @@ jobs: fi - restore_cache: keys: - - build-v2-{{ .Branch }}-{{ epoch }} - - build-v2-{{ .Branch }}- - - build-v2-master- - - build-v2- + - build-v3-{{ .Branch }}-{{ epoch }} + - build-v3-{{ .Branch }}- + - build-v3-master- + - build-v3- paths: - /tmp/docker - /tmp/images @@ -573,10 +573,10 @@ jobs: fi - restore_cache: keys: - - build-v2-{{ .Branch }}-{{ epoch }} - - build-v2-{{ .Branch }}- - - build-v2-master- - - build-v2- + - build-v3-{{ .Branch }}-{{ epoch }} + - build-v3-{{ .Branch }}- + - build-v3-master- + - build-v3- paths: - /tmp/docker - /tmp/images @@ -815,10 +815,10 @@ jobs: steps: - restore_cache: keys: - - build-v2-{{ .Branch }}-{{ epoch }} - - build-v2-{{ .Branch }}- - - build-v2-master- - - build-v2- + - build-v3-{{ .Branch }}-{{ epoch }} + - build-v3-{{ .Branch }}- + - build-v3-master- + - build-v3- paths: - /tmp/docker - run: diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml new file mode 100644 index 0000000000..b9152f03ce --- /dev/null +++ b/.github/workflows/pythonpackage.yml @@ -0,0 +1,98 @@ +name: Python package + +on: + push: + branches: [ '*' ] + tags: [ '*' ] + pull_request: + branches: [ master, 'maint/*' ] + +jobs: + build: + if: "!contains(github.event.head_commit.message, '[skip ci]')" + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.7, 3.8, 3.9] + pip: ["pip>=20.3"] + + steps: + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - uses: actions/checkout@v2 + with: + ssh-key: "${{ secrets.NIPREPS_DEPLOY }}" + - name: Fetch all tags (for versioneer to work) + if: "!startsWith(github.ref, 'refs/tags/')" + run: | + /usr/bin/git -c protocol.version=2 fetch --tags --prune --unshallow origin + - name: Build in confined, updated environment and interpolate version + run: | + python -m venv /tmp/buildenv + source /tmp/buildenv/bin/activate + python -m pip install -U setuptools pip wheel twine docutils + python setup.py sdist bdist_wheel + python -m twine check dist/smriprep* + # Interpolate version + if [[ "$GITHUB_REF" == refs/tags/* ]]; then + TAG=${GITHUB_REF##*/} + fi + THISVERSION=$( python get_version.py ) + THISVERSION=${TAG:-$THISVERSION} + echo "Expected VERSION: \"${THISVERSION}\"" + echo "THISVERSION=${THISVERSION}" >> ${GITHUB_ENV} + - name: Install in confined environment [sdist] + run: | + python -m venv /tmp/install_sdist + source /tmp/install_sdist/bin/activate + python -m pip install "${{ matrix.pip }}" setuptools + python -m pip install dist/smriprep*.tar.gz + INSTALLED_VERSION=$(python -c 'import smriprep; print(smriprep.__version__, end="")') + echo "VERSION: \"${THISVERSION}\"" + echo "INSTALLED: \"${INSTALLED_VERSION}\"" + test "${INSTALLED_VERSION}" = "${THISVERSION}" + - name: Install in confined environment [wheel] + run: | + python -m venv /tmp/install_wheel + source /tmp/install_wheel/bin/activate + python -m pip install "${{ matrix.pip }}" setuptools + python -m pip install dist/smriprep*.whl + INSTALLED_VERSION=$(python -c 'import smriprep; print(smriprep.__version__, end="")') + echo "INSTALLED: \"${INSTALLED_VERSION}\"" + test "${INSTALLED_VERSION}" = "${THISVERSION}" + - name: Install in confined environment [setup.py - install] + run: | + python -m venv /tmp/setup_install + source /tmp/setup_install/bin/activate + python -m pip install "${{ matrix.pip }}" setuptools + python -m pip install numpy scipy "Cython >= 0.28.5" # sklearn needs this + python -m pip install scikit-learn # otherwise it attempts to build it + python setup.py install + INSTALLED_VERSION=$(python -c 'import smriprep; print(smriprep.__version__, end="")') + echo "INSTALLED: \"${INSTALLED_VERSION}\"" + test "${INSTALLED_VERSION}" = "${THISVERSION}" + - name: Install in confined environment [setup.py - develop] + run: | + python -m venv /tmp/setup_develop + source /tmp/setup_develop/bin/activate + python -m pip install "${{ matrix.pip }}" setuptools + # sklearn needs these dependencies + python -m pip install numpy scipy "Cython >= 0.28.5" + python -m pip install scikit-learn # otherwise it attempts to build it + python setup.py develop + INSTALLED_VERSION=$(python -c 'import smriprep; print(smriprep.__version__, end="")') + echo "INSTALLED: \"${INSTALLED_VERSION}\"" + test "${INSTALLED_VERSION}" = "${THISVERSION}" + flake8: + if: "!contains(github.event.head_commit.message, '[skip ci]')" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + - run: pip install flake8 + - run: flake8 smriprep diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 8608bd00b7..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,61 +0,0 @@ -os: linux -dist: bionic -language: python - -cache: - directories: - - $HOME/.cache/pip - -python: - - 3.6 - - 3.7 - - 3.8 - -env: - global: - - CHECK_TYPE="test" - - INSTALL_DEPENDS="pip setuptools" - -matrix: - include: - - python: 3.6 - env: CHECK_TYPE="style" - -before_install: - - python -m pip install --upgrade virtualenv - - python -m virtualenv --python=python /tmp/venv - - source /tmp/venv/bin/activate - - which python - - python --version - - python -m pip --version - - python -m pip install --upgrade $INSTALL_DEPENDS - - python -m pip --version - -install: - - travis_retry python -m pip install . - - | - INTENDED_VERSION="$(python -c 'import versioneer; print(versioneer.get_version())')" - pushd /tmp - INSTALLED_VERSION="$(python -c 'import smriprep; print(smriprep.__version__)')" - python -c 'import smriprep; print(smriprep.__file__)' - echo "Intended: $INTENDED_VERSION" - echo "Installed: $INSTALLED_VERSION" - test "$INTENDED_VERSION" == "$INSTALLED_VERSION" - popd - -before_script: - - travis_retry python -m pip install .[$CHECK_TYPE] - -script: - - | - if [ "$CHECK_TYPE" == "style" ]; then - flake8 smriprep - elif [ "$CHECK_TYPE" == "test" ]; then - pytest -v --cov smriprep --cov-report xml:cov.xml smriprep - else - false - fi - -after_script: - - python -m pip install codecov - - codecov diff --git a/Dockerfile b/Dockerfile index ab18e487e6..30fcb23519 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,32 +1,22 @@ # Use Ubuntu 16.04 LTS -FROM ubuntu:xenial-20161213 - -# Pre-cache neurodebian key -COPY docker/files/neurodebian.gpg /usr/local/etc/neurodebian.gpg +FROM ubuntu:xenial-20201030 # Prepare environment RUN apt-get update && \ apt-get install -y --no-install-recommends \ - curl \ + autoconf \ + build-essential \ bzip2 \ ca-certificates \ - xvfb \ + curl \ cython3 \ - build-essential \ - autoconf \ + git \ libtool \ + lsb-release \ pkg-config \ - git && \ - curl -sL https://deb.nodesource.com/setup_10.x | bash - && \ - apt-get install -y --no-install-recommends \ - nodejs && \ + xvfb && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -# Install latest pandoc -RUN curl -o pandoc-2.2.2.1-1-amd64.deb -sSL "https://github.com/jgm/pandoc/releases/download/2.2.2.1/pandoc-2.2.2.1-1-amd64.deb" && \ - dpkg -i pandoc-2.2.2.1-1-amd64.deb && \ - rm pandoc-2.2.2.1-1-amd64.deb - # Installing freesurfer RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.1/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.1.tar.gz | tar zxv --no-same-owner -C /opt \ --exclude='freesurfer/diffusion' \ @@ -65,6 +55,8 @@ ENV PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \ MNI_PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \ PATH="$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH" +# Pre-cache neurodebian key +COPY docker/files/neurodebian.gpg /usr/local/etc/neurodebian.gpg # Installing Neurodebian packages (FSL, AFNI, git) RUN curl -sSL "http://neuro.debian.net/lists/$( lsb_release -c | cut -f2 ).us-ca.full" >> /etc/apt/sources.list.d/neurodebian.sources.list && \ apt-key add /usr/local/etc/neurodebian.gpg && \ @@ -74,8 +66,7 @@ RUN apt-get update && \ apt-get install -y --no-install-recommends \ fsl-core=5.0.9-5~nd16.04+1 \ afni=16.2.07~dfsg.1-5~nd16.04+1 \ - convert3d \ - git-annex-standalone && \ + convert3d && \ apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* ENV FSLDIR="/usr/share/fsl/5.0" \ @@ -91,16 +82,14 @@ ENV FSLDIR="/usr/share/fsl/5.0" \ AFNI_PLUGINPATH="/usr/lib/afni/plugins" ENV PATH="/usr/lib/fsl/5.0:/usr/lib/afni/bin:$PATH" -# Installing ANTs 2.3.4 (NeuroDocker build) +# Installing ANTs 2.3.3 (NeuroDocker build) +# Note: the URL says 2.3.4 but it is actually 2.3.3 ENV ANTSPATH=/usr/lib/ants RUN mkdir -p $ANTSPATH && \ curl -sSL "https://dl.dropbox.com/s/gwf51ykkk5bifyj/ants-Linux-centos6_x86_64-v2.3.4.tar.gz" \ | tar -xzC $ANTSPATH --strip-components 1 ENV PATH=$ANTSPATH:$PATH -# Installing SVGO -RUN npm install -g svgo - # Installing and setting up miniconda RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \ bash Miniconda3-4.5.11-Linux-x86_64.sh -b -p /usr/local/miniconda && \ @@ -114,19 +103,24 @@ ENV PATH="/usr/local/miniconda/bin:$PATH" \ PYTHONNOUSERSITE=1 # Installing precomputed python packages -RUN conda install -y python=3.7.1 \ - mkl=2018.0.3 \ - mkl-service \ - numpy=1.15.4 \ - scipy=1.1.0 \ - scikit-learn=0.19.1 \ - matplotlib=2.2.2 \ - pandas=0.23.4 \ +RUN conda install -y -c anaconda -c conda-forge \ + python=3.7.1 \ + graphviz=2.40 \ + git-annex \ libxml2=2.9.8 \ libxslt=1.1.32 \ - graphviz=2.40.1 \ + matplotlib=2.2 \ + mkl-service \ + mkl \ + nodejs \ + numpy=1.20 \ + pandas=0.23 \ + pandoc=2.11 \ + pip=20.3 \ + scikit-learn=0.19 \ + scipy=1.5 \ + setuptools=51.1 \ traits=4.6.0 \ - pip=19.1 \ zlib; sync && \ chmod -R a+rX /usr/local/miniconda; sync && \ chmod +x /usr/local/miniconda/bin/*; sync && \ @@ -147,6 +141,15 @@ RUN useradd -m -s /bin/bash -G users smriprep WORKDIR /home/smriprep ENV HOME="/home/smriprep" +# Installing SVGO +RUN npm install -g svgo + +# Installing bids-validator +RUN npm install -g bids-validator@1.4.0 + +# Refresh linked libraries +RUN ldconfig + # Installing dev requirements (packages that are not in pypi) WORKDIR /src/ @@ -164,7 +167,6 @@ RUN find $HOME -type d -exec chmod go=u {} + && \ ENV IS_DOCKER_8395080871=1 -RUN ldconfig WORKDIR /tmp/ ENTRYPOINT ["/usr/local/miniconda/bin/smriprep"] diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..501071c083 --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +.PHONY: help docker +.DEFAULT: help + +tag="latest" + +help: + @echo "Premade recipes" + @echo + @echo "make docker [tag=TAG]" + @echo "\tBuilds a docker image from source. Defaults to 'latest' tag." + + +docker: + docker build --rm -t nipreps/smriprep:$(tag) \ + --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` \ + --build-arg VCS_REF=`git rev-parse --short HEAD` \ + --build-arg VERSION=`python setup.py --version` . +