maint: refactor Dockerfile and move tests from travis to GHA

nipreps · Feb 9, 2021 · 310f76a · 310f76a
1 parent fb36542
commit 310f76a
Show file tree

Hide file tree

Showing 5 changed files with 176 additions and 119 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -44,10 +44,10 @@ jobs:
     steps:
       - restore_cache:
           keys:
-            - build-v2-{{ .Branch }}-{{ epoch }}
-            - build-v2-{{ .Branch }}-
-            - build-v2-master-
-            - build-v2-
+            - build-v3-{{ .Branch }}-{{ epoch }}
+            - build-v3-{{ .Branch }}-
+            - build-v3-master-
+            - build-v3-
           paths:
             - /tmp/docker
             - /tmp/images
@@ -80,14 +80,14 @@ jobs:
             set -e
             if [[ "$success" = "0" ]]; then
                 echo "Pulling from local registry"
-                docker tag localhost:5000/ubuntu ubuntu:xenial-20161213
+                docker tag localhost:5000/ubuntu ubuntu:xenial-20201030
                 docker pull localhost:5000/smriprep
                 docker tag localhost:5000/smriprep nipreps/smriprep:latest
                 docker tag localhost:5000/smriprep nipreps/smriprep
             else
                 echo "Pulling from Docker Hub"
-                docker pull ubuntu:xenial-20161213
-                docker tag ubuntu:xenial-20161213 localhost:5000/ubuntu
+                docker pull ubuntu:xenial-20201030
+                docker tag ubuntu:xenial-20201030 localhost:5000/ubuntu
                 docker push localhost:5000/ubuntu
                 docker pull nipreps/smriprep:latest
             fi
@@ -130,7 +130,7 @@ jobs:
             docker exec -it registry /bin/registry garbage-collect --delete-untagged \
                 /etc/docker/registry/config.yml
       - save_cache:
-         key: build-v2-{{ .Branch }}-{{ epoch }}
+         key: build-v3-{{ .Branch }}-{{ epoch }}
          paths:
             - /tmp/docker
             - /tmp/images
@@ -241,10 +241,10 @@ jobs:
           at: /tmp
       - restore_cache:
           keys:
-            - build-v2-{{ .Branch }}-{{ epoch }}
-            - build-v2-{{ .Branch }}-
-            - build-v2-master-
-            - build-v2-
+            - build-v3-{{ .Branch }}-{{ epoch }}
+            - build-v3-{{ .Branch }}-
+            - build-v3-master-
+            - build-v3-
           paths:
             - /tmp/docker
             - /tmp/images
@@ -390,10 +390,10 @@ jobs:
             fi
       - restore_cache:
           keys:
-            - build-v2-{{ .Branch }}-{{ epoch }}
-            - build-v2-{{ .Branch }}-
-            - build-v2-master-
-            - build-v2-
+            - build-v3-{{ .Branch }}-{{ epoch }}
+            - build-v3-{{ .Branch }}-
+            - build-v3-master-
+            - build-v3-
           paths:
             - /tmp/docker
             - /tmp/images
@@ -573,10 +573,10 @@ jobs:
             fi
       - restore_cache:
           keys:
-            - build-v2-{{ .Branch }}-{{ epoch }}
-            - build-v2-{{ .Branch }}-
-            - build-v2-master-
-            - build-v2-
+            - build-v3-{{ .Branch }}-{{ epoch }}
+            - build-v3-{{ .Branch }}-
+            - build-v3-master-
+            - build-v3-
           paths:
             - /tmp/docker
             - /tmp/images
@@ -815,10 +815,10 @@ jobs:
     steps:
       - restore_cache:
           keys:
-            - build-v2-{{ .Branch }}-{{ epoch }}
-            - build-v2-{{ .Branch }}-
-            - build-v2-master-
-            - build-v2-
+            - build-v3-{{ .Branch }}-{{ epoch }}
+            - build-v3-{{ .Branch }}-
+            - build-v3-master-
+            - build-v3-
           paths:
             - /tmp/docker
       - run:

diff --git a/.github/workflows/pythonpackage.yml b/.github/workflows/pythonpackage.yml
@@ -0,0 +1,98 @@
+name: Python package
+
+on:
+  push:
+    branches: [ '*' ]
+    tags: [ '*' ]
+  pull_request:
+    branches: [ master, 'maint/*' ]
+
+jobs:
+  build:
+    if: "!contains(github.event.head_commit.message, '[skip ci]')"
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.7, 3.8, 3.9]
+        pip: ["pip>=20.3"]
+
+    steps:
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - uses: actions/checkout@v2
+      with:
+        ssh-key: "${{ secrets.NIPREPS_DEPLOY }}"
+    - name: Fetch all tags (for versioneer to work)
+      if: "!startsWith(github.ref, 'refs/tags/')"
+      run: |
+        /usr/bin/git -c protocol.version=2 fetch --tags --prune --unshallow origin
+    - name: Build in confined, updated environment and interpolate version
+      run: |
+        python -m venv /tmp/buildenv
+        source /tmp/buildenv/bin/activate
+        python -m pip install -U setuptools pip wheel twine docutils
+        python setup.py sdist bdist_wheel
+        python -m twine check dist/smriprep*
+        # Interpolate version
+        if [[ "$GITHUB_REF" == refs/tags/* ]]; then
+          TAG=${GITHUB_REF##*/}
+        fi
+        THISVERSION=$( python get_version.py )
+        THISVERSION=${TAG:-$THISVERSION}
+        echo "Expected VERSION: \"${THISVERSION}\""
+        echo "THISVERSION=${THISVERSION}" >> ${GITHUB_ENV}
+    - name: Install in confined environment [sdist]
+      run: |
+        python -m venv /tmp/install_sdist
+        source /tmp/install_sdist/bin/activate
+        python -m pip install "${{ matrix.pip }}" setuptools
+        python -m pip install dist/smriprep*.tar.gz
+        INSTALLED_VERSION=$(python -c 'import smriprep; print(smriprep.__version__, end="")')
+        echo "VERSION: \"${THISVERSION}\""
+        echo "INSTALLED: \"${INSTALLED_VERSION}\""
+        test "${INSTALLED_VERSION}" = "${THISVERSION}"
+    - name: Install in confined environment [wheel]
+      run: |
+        python -m venv /tmp/install_wheel
+        source /tmp/install_wheel/bin/activate
+        python -m pip install "${{ matrix.pip }}" setuptools
+        python -m pip install dist/smriprep*.whl
+        INSTALLED_VERSION=$(python -c 'import smriprep; print(smriprep.__version__, end="")')
+        echo "INSTALLED: \"${INSTALLED_VERSION}\""
+        test "${INSTALLED_VERSION}" = "${THISVERSION}"
+    - name: Install in confined environment [setup.py - install]
+      run: |
+        python -m venv /tmp/setup_install
+        source /tmp/setup_install/bin/activate
+        python -m pip install "${{ matrix.pip }}" setuptools
+        python -m pip install numpy scipy "Cython >= 0.28.5" # sklearn needs this
+        python -m pip install scikit-learn  # otherwise it attempts to build it
+        python setup.py install
+        INSTALLED_VERSION=$(python -c 'import smriprep; print(smriprep.__version__, end="")')
+        echo "INSTALLED: \"${INSTALLED_VERSION}\""
+        test "${INSTALLED_VERSION}" = "${THISVERSION}"
+    - name: Install in confined environment [setup.py - develop]
+      run: |
+        python -m venv /tmp/setup_develop
+        source /tmp/setup_develop/bin/activate
+        python -m pip install "${{ matrix.pip }}" setuptools
+        # sklearn needs these dependencies
+        python -m pip install numpy scipy "Cython >= 0.28.5"
+        python -m pip install scikit-learn  # otherwise it attempts to build it
+        python setup.py develop
+        INSTALLED_VERSION=$(python -c 'import smriprep; print(smriprep.__version__, end="")')
+        echo "INSTALLED: \"${INSTALLED_VERSION}\""
+        test "${INSTALLED_VERSION}" = "${THISVERSION}"
+  flake8:
+    if: "!contains(github.event.head_commit.message, '[skip ci]')"
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.7
+      uses: actions/setup-python@v1
+      with:
+        python-version: 3.7
+    - run: pip install flake8
+    - run: flake8 smriprep
diff --git a/.travis.yml b/.travis.yml
diff --git a/Dockerfile b/Dockerfile
@@ -1,32 +1,22 @@
 # Use Ubuntu 16.04 LTS
-FROM ubuntu:xenial-20161213
-
-# Pre-cache neurodebian key
-COPY docker/files/neurodebian.gpg /usr/local/etc/neurodebian.gpg
+FROM ubuntu:xenial-20201030
 
 # Prepare environment
 RUN apt-get update && \
     apt-get install -y --no-install-recommends \
-                    curl \
+                    autoconf \
+                    build-essential \
                     bzip2 \
                     ca-certificates \
-                    xvfb \
+                    curl \
                     cython3 \
-                    build-essential \
-                    autoconf \
+                    git \
                     libtool \
+                    lsb-release \
                     pkg-config \
-                    git && \
-    curl -sL https://deb.nodesource.com/setup_10.x | bash - && \
-    apt-get install -y --no-install-recommends \
-                    nodejs && \
+                    xvfb && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-# Install latest pandoc
-RUN curl -o pandoc-2.2.2.1-1-amd64.deb -sSL "https://github.com/jgm/pandoc/releases/download/2.2.2.1/pandoc-2.2.2.1-1-amd64.deb" && \
-    dpkg -i pandoc-2.2.2.1-1-amd64.deb && \
-    rm pandoc-2.2.2.1-1-amd64.deb
-
 # Installing freesurfer
 RUN curl -sSL https://surfer.nmr.mgh.harvard.edu/pub/dist/freesurfer/6.0.1/freesurfer-Linux-centos6_x86_64-stable-pub-v6.0.1.tar.gz | tar zxv --no-same-owner -C /opt \
     --exclude='freesurfer/diffusion' \
@@ -65,6 +55,8 @@ ENV PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \
     MNI_PERL5LIB="$MINC_LIB_DIR/perl5/5.8.5" \
     PATH="$FREESURFER_HOME/bin:$FSFAST_HOME/bin:$FREESURFER_HOME/tktools:$MINC_BIN_DIR:$PATH"
 
+# Pre-cache neurodebian key
+COPY docker/files/neurodebian.gpg /usr/local/etc/neurodebian.gpg
 # Installing Neurodebian packages (FSL, AFNI, git)
 RUN curl -sSL "http://neuro.debian.net/lists/$( lsb_release -c | cut -f2 ).us-ca.full" >> /etc/apt/sources.list.d/neurodebian.sources.list && \
     apt-key add /usr/local/etc/neurodebian.gpg && \
@@ -74,8 +66,7 @@ RUN apt-get update && \
     apt-get install -y --no-install-recommends \
                     fsl-core=5.0.9-5~nd16.04+1 \
                     afni=16.2.07~dfsg.1-5~nd16.04+1 \
-                    convert3d \
-                    git-annex-standalone && \
+                    convert3d && \
     apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 ENV FSLDIR="/usr/share/fsl/5.0" \
@@ -91,16 +82,14 @@ ENV FSLDIR="/usr/share/fsl/5.0" \
     AFNI_PLUGINPATH="/usr/lib/afni/plugins"
 ENV PATH="/usr/lib/fsl/5.0:/usr/lib/afni/bin:$PATH"
 
-# Installing ANTs 2.3.4 (NeuroDocker build)
+# Installing ANTs 2.3.3 (NeuroDocker build)
+# Note: the URL says 2.3.4 but it is actually 2.3.3
 ENV ANTSPATH=/usr/lib/ants
 RUN mkdir -p $ANTSPATH && \
     curl -sSL "https://dl.dropbox.com/s/gwf51ykkk5bifyj/ants-Linux-centos6_x86_64-v2.3.4.tar.gz" \
     | tar -xzC $ANTSPATH --strip-components 1
 ENV PATH=$ANTSPATH:$PATH
 
-# Installing SVGO
-RUN npm install -g svgo
-
 # Installing and setting up miniconda
 RUN curl -sSLO https://repo.continuum.io/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh && \
     bash Miniconda3-4.5.11-Linux-x86_64.sh -b -p /usr/local/miniconda && \
@@ -114,19 +103,24 @@ ENV PATH="/usr/local/miniconda/bin:$PATH" \
     PYTHONNOUSERSITE=1
 
 # Installing precomputed python packages
-RUN conda install -y python=3.7.1 \
-                     mkl=2018.0.3 \
-                     mkl-service \
-                     numpy=1.15.4 \
-                     scipy=1.1.0 \
-                     scikit-learn=0.19.1 \
-                     matplotlib=2.2.2 \
-                     pandas=0.23.4 \
+RUN conda install -y -c anaconda -c conda-forge \
+                     python=3.7.1 \
+                     graphviz=2.40 \
+                     git-annex \
                      libxml2=2.9.8 \
                      libxslt=1.1.32 \
-                     graphviz=2.40.1 \
+                     matplotlib=2.2 \
+                     mkl-service \
+                     mkl \
+                     nodejs \
+                     numpy=1.20 \
+                     pandas=0.23 \
+                     pandoc=2.11 \
+                     pip=20.3 \
+                     scikit-learn=0.19 \
+                     scipy=1.5 \
+                     setuptools=51.1 \
                      traits=4.6.0 \
-                     pip=19.1 \
                      zlib; sync && \
     chmod -R a+rX /usr/local/miniconda; sync && \
     chmod +x /usr/local/miniconda/bin/*; sync && \
@@ -147,6 +141,15 @@ RUN useradd -m -s /bin/bash -G users smriprep
 WORKDIR /home/smriprep
 ENV HOME="/home/smriprep"
 
+# Installing SVGO
+RUN npm install -g svgo
+
+# Installing bids-validator
+RUN npm install -g [email protected]
+
+# Refresh linked libraries
+RUN ldconfig
+
 # Installing dev requirements (packages that are not in pypi)
 WORKDIR /src/
 
@@ -164,7 +167,6 @@ RUN find $HOME -type d -exec chmod go=u {} + && \
 
 ENV IS_DOCKER_8395080871=1
 
-RUN ldconfig
 WORKDIR /tmp/
 ENTRYPOINT ["/usr/local/miniconda/bin/smriprep"]