Merge pull request #187 from Autodesk/publishing_fixes

Publishing fixes

.coveragerc

@@ -1,7 +1,7 @@
 [paths]
 source =
   moldesign
-
+  */moldesign
 
 [run]
 source = moldesign

DockerMakefiles/Moldesign.yml

@@ -5,7 +5,6 @@ moldesign_py_build:
   requires:
    - python_deploy
   build_directory: ..
-  ignorefile: buildfiles/moldesign/moldesign.dockerignore
   build: |
     COPY . /opt/molecular-design-toolkit
     WORKDIR /opt/molecular-design-toolkit

codeship-services.yml

@@ -2,16 +2,27 @@ image_builder: &build-base
   build:
     context: .
     dockerfile: ./deployment/build-env.dockerfile
-  volumes:
-    - ./tmp/dist:/opt/molecular-design-toolkit/dist
   add_docker: true
   cached: true
   working_dir: /opt/molecular-design-toolkit
+  environment: &repo
+    REPO: "docker.io/autodesk/moldesign:"
+
+
+coverage_pusher:
+   <<: *build-base
+   cached: false
+   encrypted_env_file: ./deployment/test-tokens.crypt
+   volumes:
+    - ./tmp/reports:/opt/reports
 
 
 publisher:
    <<: *build-base
+   cached: false
    encrypted_env_file: ./deployment/deploy-tokens.crypt
+   volumes:
+    - ./tmp/dist:/opt/dist
 
 
 test_moldesign_minimal: &test-base
@@ -26,6 +37,7 @@ test_moldesign_minimal: &test-base
     args:
       baseimage: moldesign_minimal
   environment:
+    <<: *repo
     TESTENV: minimal
     PYVERSION: 3
     OPENMM_CPU_THREADS: 1
@@ -38,6 +50,7 @@ test_moldesign_minimal_py2:
     args:
       baseimage: moldesign_minimal_py2
   environment:
+    <<: *repo
     TESTENV: minimal
     PYVERSION: 2
     OPENMM_CPU_THREADS: 1
@@ -50,6 +63,7 @@ test_moldesign_complete:
     args:
       baseimage: moldesign_complete
   environment:
+    <<: *repo
     TESTENV: complete
     PYVERSION: 3
     OPENMM_CPU_THREADS: 1
@@ -62,6 +76,7 @@ test_moldesign_complete_py2:
     args:
       baseimage: moldesign_complete_py2
   environment:
+    <<: *repo
     TESTENV: complete
     PYVERSION: 2
     OPENMM_CPU_THREADS: 1

codeship-steps.yml

@@ -1,63 +1,72 @@
+# Pull previous builds from DockerHub to use as a cache.
+# These images are tagged as [imagename]-cache
 - name: pull-cache
   service: image_builder
   command: deployment/pull-cache.sh
 
+# Build the python package to be tested (and uploaded to PyPI if this is a release)
+# The package is built in the "autodesk/moldesign:moldesign_py_build-[branch name]" image
 - name: build-sdist
   service: image_builder
-  command: docker-make moldesign_py_build
+  command: bash -c "docker-make moldesign_py_build
+            --tag dev
             --keep-build-tags
-            -f DockerMakefiles/DockerMake.yml --tag dev
+            -f DockerMakefiles/DockerMake.yml"
 
+# Build docker images for all test environments. They will be tagged as
+# [imagename]-dev
+# (For the sake of parallelism, we also download other images that the tests depend on)
 - name: build-and-pull-images
   service: image_builder
   type: parallel
   steps:
   - name: pull-dependent-images
     command: deployment/pull-chemdocker.sh
   - name: build_py3
-    command:
-         docker-make moldesign_minimal moldesign_complete
+    command: bash -c
+          "docker-make moldesign_minimal moldesign_complete
           -f DockerMakefiles/DockerMake.yml
-          --tag dev --keep-build-tags
-          --cache-repo moldesign --cache-tag cache
+          --tag dev
+          --keep-build-tags
+          --cache-tag cache"
   - name: build_py2
-    command:
-        docker-make moldesign_minimal_py2 moldesign_complete_py2
+    command: bash -c
+         "docker-make moldesign_minimal_py2 moldesign_complete_py2
          -f DockerMakefiles/DockerMake.yml
-         --tag dev --keep-build-tags
-         --cache-repo moldesign --cache-tag cache
+         --tag dev
+         --keep-build-tags
+         --cache-tag cache"
 
-- name: push-images
+
+# Push build artifacts to Dockerhub.
+# These are tagged as autodesk/moldesign:[imagename]-[branchame]-devbuild
+# For testing purposes, they are also tagged as
+# autodesk/moldesign:[imagename]-[branchname] (this is the tag that will be pushed
+# to dockerhub for successful releases)
+- name: tag-and-push-images
   type: parallel
   steps:
     - name: push-artifacts
-      command: deployment/push.sh moldesign_minimal moldesign_py2
-                                  moldesign_complete moldesign_complete_py2
+      command: deployment/push-and-tag.sh
+          moldesign_minimal moldesign_minimal_py2 moldesign_py_build
+          moldesign_complete moldesign_complete_py2
       service: publisher
     - type: serial
       name: make-notebook
       steps:
         - name: build-notebook
           service: image_builder
-          command: docker-make moldesign_notebook
+          command: bash -c
+              "docker-make moldesign_notebook
               -f DockerMakefiles/DockerMake.yml
-              --tag dev --cache-repo moldesign --cache-tag cache
+              --tag dev
+              --cache-tag cache"
         - name: push-notebook
           service: publisher
-          command: deployment/push.sh moldesign_notebook
-
+          command: deployment/push-and-tag.sh moldesign_notebook
 
-- name: print-environments
-  type: parallel
-  services:
-    - test_moldesign_complete
-    - test_moldesign_complete_py2
-    - test_moldesign_minimal
-    - test_moldesign_minimal_py2
-  steps:
-   - command: deployment/print-environment.sh
-     name: env-report
 
+# Run the tests in each environment
 - name: run-tests
   type: serial
   services:
@@ -66,12 +75,33 @@
    - test_moldesign_minimal_py2
    - test_moldesign_minimal
   steps:
+   - command: deployment/print-environment.sh
+     name: environment
    - command: deployment/run-ci-tests.sh
-     name: testrunner
+     name: tests
+
+
+# Upload coverage results
+- name: push-coverage
+  service: coverage_pusher
+  command: deployment/push-coverage.sh
 
-- name: publish-python
-  service: publisher
+
+# If this build is tagged with a PEP440-compliant version number AND the tests have passed,
+# upload the package to PyPI, and push the docker images to dockerhub as
+# autodesk/moldesign:[imagename]-[version-number]
+- name: release
   # matches tags that are valid PEP440 versions
+  type: serial
   tag: '^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)((a|rc|b)(0|[1-9]\d*))?$'
-  command: deployment/publish.sh
+  steps:
+   - type: parallel
+     services:
+       - test_moldesign_complete
+       - test_moldesign_complete_py2
+     steps:
+     - command: deployment/test-version-number.sh
+   - name: upload
+     command: deployment/publish.sh
+     service: publisher
 

deployment/README.md

@@ -1 +1,5 @@
-Files in this directory are for use with our Codeship CI/CD setup, and aren't generally useful outside of that context.
+Files in this directory are for use with our Codeship CI/CD setup,
+and aren't generally useful outside of that context.
+
+
+The build process is annotated in ../codeship-steps.yml

deployment/build-env.dockerfile

@@ -10,6 +10,8 @@ RUN curl -fsSLO https://get.docker.com/builds/Linux/x86_64/docker-17.04.0-ce.tgz
   && mv docker/docker /usr/local/bin \
   && rm -r docker docker-17.04.0-ce.tgz
 
-ADD . /opt/molecular-design-toolkit
-RUN pip install -r /opt/molecular-design-toolkit/DockerMakefiles/requirements.txt
+ADD ./DockerMakefiles/requirements.txt /tmp/reqs.txt
+RUN pip install -r /tmp/reqs.txt \
+ && pip install coveralls twine
 
+ADD . /opt/molecular-design-toolkit

deployment/publish.sh

@@ -8,18 +8,25 @@
 # fail immediately if any command fails:
 set -e
 
-pyversion=$(python -m moldesign version | tail -n 1)
+echo "Now deploying moldesign-${CI_BRANCH}"
 
-if [ "${pyversion}" == "${CI_BRANCH}" ]; then
-  echo "Deploying version ${CI_BRANCH}"
-else
-  echo "Can't publish - moldesign package version '${pyversion}' differs from its Git tag '${CI_BRANCH}'"
-  exit 1
-fi
+docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_PASSWORD}
 
 
-# Copy build artifacts
-sdist=moldesign-${pyversion}.tar.gz
-docker run moldesign_py_build:dev -v ./tmp/dists:/hostdists  cp dist/${sdist} /hostdists
+# Copy python package out of the docker image
+sdist=moldesign-${CI_BRANCH}.tar.gz
+docker run moldesign_py_build:dev cat dist/${sdist} > /opt/dist/${sdist}
+
+# Push images to dockerhub
+for img in moldesign_minimal       \
+           moldesign_minimal_py2   \
+           moldesign_complete      \
+           moldesign_complete_py2  \
+           moldesign_notebook; do
+   docker push ${REPO}${img}-${CI_BRANCH} | tee -a push.log | egrep -i 'pull|already'
+done
+
+
+# Push python package to PyPI
 echo "Uploading version ${CI_BRANCH} to PyPI:"
-twine upload -u ${PYPI_USER} -p ${PYPI_PASSWORD} ./tmp/dists/moldesign-${pyversion}.tar.gz
+twine upload -u ${PYPI_USER} -p ${PYPI_PASSWORD} /opt/dist/${sdist}

deployment/pull-cache.sh

@@ -14,15 +14,15 @@ function run-pull(){
     # pull an image. If successful, retags the image with the "cache" tag
     img=$1
     tag=$2
-    imgpath="autodesk/moldesign:${img}-${tag}"
+    imgpath="${REPO}${img}-${tag}"
 
     echocmd docker pull ${imgpath} | tee -a pull.log | egrep -i 'pull|already';
 
     success=${PIPESTATUS[0]}
     if [ "$success" -ne 0 ]; then
        return ${success};
     else
-       docker tag ${imgpath} moldesign/${img}:cache
+       docker tag ${imgpath} ${img}:cache
     fi
 }
 
@@ -34,6 +34,7 @@ for img in moldesign_minimal       \
            moldesign_minimal_py2   \
            moldesign_complete      \
            moldesign_complete_py2  \
+           moldesign_py_build      \
            moldesign_notebook; do
    run-pull ${img} ${CI_BRANCH}-devbuild || run-pull ${img} master || \
    echo " --> Failed to pull cache for ${img}"

deployment/push.sh → deployment/push-and-tag.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 if [ -z ${CI_BRANCH} ]; then
   echo "\$CI_BRANCH" var not set.
   exit 1
@@ -18,8 +20,11 @@ function echocmd() {
 docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_PASSWORD}
 
 for img in $@; do
-    remote_img=autodesk/moldesign:${img}-${CI_BRANCH}-devbuild
+    build_img=${img}:dev
+    release_tag=${REPO}${img}-${CI_BRANCH}
+    artifact_tag=${release_tag}-devbuild
 
-    echocmd docker tag ${img}:dev ${remote_img}
-    echocmd docker push ${remote_img} | tee -a push.log | egrep -i 'push|already';
+    echocmd docker tag ${build_img} ${release_tag}
+    echocmd docker tag ${build_img} ${artifact_tag}
+    echocmd docker push ${artifact_tag} | tee -a push.log | egrep -i 'push|already';
 done

deployment/push-coverage.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# combines coverage from all test environments and pushes it to coveralls.io
+
+set -e
+
+mkdir -p /opt/reports/env-coverage
+if $(cp /opt/reports/env-coverage/coverage.* /opt/reports/); then
+  coverage combine /opt/reports/coverage.*
+  coveralls || echo "Failed to upload coverage to coveralls.io"
+else
+   echo "No coverage files found, skipping coveralls upload"
+fi