diff --git a/codeship-services.yml b/codeship-services.yml index ab5f69b..c5417bb 100644 --- a/codeship-services.yml +++ b/codeship-services.yml @@ -7,6 +7,8 @@ image_builder: &build-base add_docker: true cached: true working_dir: /opt/molecular-design-toolkit + environment: &repo + REPO: "docker.io/autodesk/moldesign:" publisher: @@ -26,6 +28,7 @@ test_moldesign_minimal: &test-base args: baseimage: moldesign_minimal environment: + <<: *repo TESTENV: minimal PYVERSION: 3 OPENMM_CPU_THREADS: 1 @@ -38,6 +41,7 @@ test_moldesign_minimal_py2: args: baseimage: moldesign_minimal_py2 environment: + <<: *repo TESTENV: minimal PYVERSION: 2 OPENMM_CPU_THREADS: 1 @@ -50,6 +54,7 @@ test_moldesign_complete: args: baseimage: moldesign_complete environment: + <<: *repo TESTENV: complete PYVERSION: 3 OPENMM_CPU_THREADS: 1 @@ -62,6 +67,7 @@ test_moldesign_complete_py2: args: baseimage: moldesign_complete_py2 environment: + <<: *repo TESTENV: complete PYVERSION: 2 OPENMM_CPU_THREADS: 1 diff --git a/codeship-steps.yml b/codeship-steps.yml index a6b8fa1..d4bfed8 100644 --- a/codeship-steps.yml +++ b/codeship-steps.yml @@ -1,13 +1,21 @@ +# Pull previous builds from DockerHub to use as a cache. +# These images are tagged as [imagename]-cache - name: pull-cache service: image_builder command: deployment/pull-cache.sh +# Build the python package to be tested (and uploaded to PyPI if this is a release) +# The package is built in the "autodesk/moldesign:moldesign_py_build-[branch name]" image - name: build-sdist service: image_builder - command: docker-make moldesign_py_build + command: bash -c "docker-make moldesign_py_build + --repo ${REPO} --tag ${CI_BRANCH} --keep-build-tags - -f DockerMakefiles/DockerMake.yml --tag dev + -f DockerMakefiles/DockerMake.yml" +# Build docker images for all test environments. They will be tagged as +# autodesk/moldesign:[imagename]-[branchname] +# (For the sake of parallelism, we also download other images that the tests depend on) - name: build-and-pull-images service: image_builder type: parallel @@ -15,23 +23,28 @@ - name: pull-dependent-images command: deployment/pull-chemdocker.sh - name: build_py3 - command: - docker-make moldesign_minimal moldesign_complete + command: bash -c + "docker-make moldesign_minimal moldesign_complete -f DockerMakefiles/DockerMake.yml - --tag dev --keep-build-tags - --cache-repo moldesign --cache-tag cache + --repo ${REPO} --tag ${CI_BRANCH} + --keep-build-tags + --cache-repo moldesign --cache-tag cache" - name: build_py2 - command: - docker-make moldesign_minimal_py2 moldesign_complete_py2 + command: bash -c + "docker-make moldesign_minimal_py2 moldesign_complete_py2 -f DockerMakefiles/DockerMake.yml - --tag dev --keep-build-tags - --cache-repo moldesign --cache-tag cache + --repo ${REPO} --tag ${CI_BRANCH} + --keep-build-tags + --cache-repo moldesign --cache-tag cache" + +# Push build artifacts to Dockerhub. +# These are tagged as autodesk/moldesign:[imagename]-[branchame]-devbuild - name: push-images type: parallel steps: - name: push-artifacts - command: deployment/push.sh moldesign_minimal moldesign_py2 + command: deployment/push.sh moldesign_minimal moldesign_py2 moldesign_py_build moldesign_complete moldesign_complete_py2 service: publisher - type: serial @@ -39,14 +52,17 @@ steps: - name: build-notebook service: image_builder - command: docker-make moldesign_notebook + command: bash -c + "docker-make moldesign_notebook -f DockerMakefiles/DockerMake.yml - --tag dev --cache-repo moldesign --cache-tag cache + --repo ${REPO} --tag ${CI_BRANCH} + --cache-tag cache" - name: push-notebook service: publisher command: deployment/push.sh moldesign_notebook +# Prints out internal information about the test environments - name: print-environments type: parallel services: @@ -58,6 +74,7 @@ - command: deployment/print-environment.sh name: env-report +# Run the tests in each environment. - name: run-tests type: serial services: @@ -69,6 +86,9 @@ - command: deployment/run-ci-tests.sh name: testrunner +# If this build is tagged with a PEP440-compliant version number AND the tests have passed, +# upload the package to PyPI, and push the docker images to dockerhub as +# autodesk/moldesign:[imagename]-[version-number] - name: publish-python service: publisher # matches tags that are valid PEP440 versions diff --git a/deployment/README.md b/deployment/README.md index 0602eb8..44507d8 100644 --- a/deployment/README.md +++ b/deployment/README.md @@ -1 +1,5 @@ -Files in this directory are for use with our Codeship CI/CD setup, and aren't generally useful outside of that context. \ No newline at end of file +Files in this directory are for use with our Codeship CI/CD setup, +and aren't generally useful outside of that context. + + +The build process is annotated in ../codeship-steps.yml \ No newline at end of file diff --git a/deployment/publish.sh b/deployment/publish.sh index 8756aaa..b10de52 100755 --- a/deployment/publish.sh +++ b/deployment/publish.sh @@ -18,6 +18,16 @@ else fi +# Push images +for img in moldesign_minimal \ + moldesign_minimal_py2 \ + moldesign_complete \ + moldesign_complete_py2 \ + moldesign_notebook; do + docker push ${REPO}{$img}-${CI_BRANCH} | tee -a push.log | egrep -i 'pull|already' +done + + # Copy build artifacts sdist=moldesign-${pyversion}.tar.gz docker run moldesign_py_build:dev -v ./tmp/dists:/hostdists cp dist/${sdist} /hostdists diff --git a/deployment/pull-cache.sh b/deployment/pull-cache.sh index 737a51c..7c35b4c 100755 --- a/deployment/pull-cache.sh +++ b/deployment/pull-cache.sh @@ -14,7 +14,7 @@ function run-pull(){ # pull an image. If successful, retags the image with the "cache" tag img=$1 tag=$2 - imgpath="autodesk/moldesign:${img}-${tag}" + imgpath="${REPO}${img}-${tag}" echocmd docker pull ${imgpath} | tee -a pull.log | egrep -i 'pull|already'; @@ -34,6 +34,7 @@ for img in moldesign_minimal \ moldesign_minimal_py2 \ moldesign_complete \ moldesign_complete_py2 \ + moldesign_py_build \ moldesign_notebook; do run-pull ${img} ${CI_BRANCH}-devbuild || run-pull ${img} master || \ echo " --> Failed to pull cache for ${img}" diff --git a/deployment/push.sh b/deployment/push.sh index 53a54e4..79bbbbb 100755 --- a/deployment/push.sh +++ b/deployment/push.sh @@ -18,8 +18,9 @@ function echocmd() { docker login -u ${DOCKERHUB_USER} -p ${DOCKERHUB_PASSWORD} for img in $@; do - remote_img=autodesk/moldesign:${img}-${CI_BRANCH}-devbuild + local_img=${REPO}${img}-${CI_BRANCH} + remote_img=${local_img}-devbuild - echocmd docker tag ${img}:dev ${remote_img} + echocmd docker tag ${local_img} ${remote_img} echocmd docker push ${remote_img} | tee -a push.log | egrep -i 'push|already'; done \ No newline at end of file diff --git a/deployment/run-ci-tests.sh b/deployment/run-ci-tests.sh index be6d0f3..de341a1 100755 --- a/deployment/run-ci-tests.sh +++ b/deployment/run-ci-tests.sh @@ -1,7 +1,19 @@ #!/usr/bin/env bash +# Drives tests for our CI system. This looks for the following environment variables: +# Defined by codeship +# - CI_BRANCH +# - CI_COMMIT_MESSAGE +# Defined in ../codeship-services.yml +# - TESTENV +# - PYVERSION set -e # fail immediately if any command fails +if [ -z "${CI_BRANCH}" ]; then + echo "FAILURE: Variable \$CI_BRANCH not defined." + exit 1 +fi + install_location=$(python -c "import moldesign, os; print(moldesign.__path__[0])") test_location=$(dirname "${install_location}") @@ -19,22 +31,31 @@ function send_status_update(){ function check_if_tests_should_run(){ echo "Should I run the tests in this environment?" - runthem=false if [[ "${CI_COMMIT_MESSAGE}" == *"--fast-ci-tests"* && "${VERSION}" != "complete.py3" ]]; then echo "NO: found \"--fast-ci-tests\" flag in commit message; run complete.py3 only" exit 0 fi + if [[ "${CI_BRANCH}" =~ ^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)((a|rc|b)(0|[1-9]\d*))?$ ]] + then + echo "YES: this is a release version: \"${CI_BRANCH}\"" + return 0 + else # otherwise, point to the appropriate docker image tag + mkdir -p ~/.moldesign + echo "default_version_tag: ${CI_BRANCH}" >> ~/.moldesign/moldesign.yml + fi + + if [ "${TESTENV}" == "complete" ]; then - runthem=true echo "YES: always run in 'complete' environment" + return 0 fi case "${CI_BRANCH}" in master|deploy|dev) - runthem=true echo "YES: always run in branch \"${CI_BRANCH}\"" + return 0 ;; esac