.gitlab-ci.yml

image: quay.io/vgteam/vg_ci_prebake:latest
# Note that we must run in a privileged container for our internal Docker daemon to come up.

before_script:
  - startdocker || true
  - docker info
  - cat /etc/hosts
  - export PYTHONIOENCODING=utf-8
  - mkdir -p ~/.kube && cp "$GITLAB_SECRET_FILE_KUBE_CONFIG" ~/.kube/config
  - mkdir -p ~/.aws && cp "$GITLAB_SECRET_FILE_AWS_CREDENTIALS" ~/.aws/credentials

after_script:
  # We need to clean up any files that Toil may have made via Docker that
  # aren't deletable by the Gitlab user. If we don't do this, Gitlab will try
  # and clean them up before running the next job on the runner, fail, and fail
  # that next job.
  - pwd
  - sudo rm -rf tmp
  - stopdocker || true

stages:
  - basic_tests
  - main_tests
  - integration


# Python3.6
py36_main:
  stage: basic_tests
  script:
    - pwd
    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor
    - make test tests=src/toil/test/src
    - make test tests=src/toil/test/utils

py36_appliance_build:
  stage: basic_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq
    - virtualenv -p python3.6 venv && . venv/bin/activate && make prepare && pip install pycparser && make develop extras=[all] && pip install htcondor awscli==1.16.272
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    - python setup_gitlab_docker.py
    - make push_docker


# Python3.7
py37_batch_systems:
  stage: main_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
    - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli==1.16.272
    - make test tests=src/toil/test/batchSystems/batchSystemTest.py
    - make test tests=src/toil/test/mesos/MesosDataStructuresTest.py

py37_cwl_v1.0:
  stage: main_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
    - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[cwl,aws]
    - mypy --ignore-missing-imports --no-strict-optional $(pwd)/src/toil/cwl/cwltoil.py  # make this a separate linting stage
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv10Test

py37_cwl_v1.1:
  stage: main_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
    - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[cwl,aws]
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv11Test

py37_cwl_v1.2:
  stage: main_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
    - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[cwl,aws]
    - make test tests=src/toil/test/cwl/cwlTest.py::CWLv12Test

py37_wdl:
  stage: main_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
    - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all]
    - make test tests=src/toil/test/wdl/toilwdlTest.py
    - make test tests=src/toil/test/wdl/builtinTest.py

py37_jobstore_and_provisioning:
  stage: main_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
    - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor
    - make test tests=src/toil/test/jobStores/jobStoreTest.py
    - make test tests=src/toil/test/sort/sortTest.py
    - make test tests=src/toil/test/provisioners/aws/awsProvisionerTest.py
    - make test tests=src/toil/test/provisioners/clusterScalerTest.py
#    - python -m pytest --duration=0 -s -r s src/toil/test/provisioners/gceProvisionerTest.py
# https://ucsc-ci.com/databiosphere/toil/-/jobs/38672
# guessing decorators are masking class as function?  ^  also, abstract class is run as normal test?  should hide.

py37_main:
  stage: basic_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
    - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor
    - make test tests=src/toil/test/src
    - make test tests=src/toil/test/utils

py37_appliance_build:
  stage: basic_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
    - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && pip install pycparser && make develop extras=[all] && pip install htcondor awscli==1.16.272
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    - python setup_gitlab_docker.py
    - make push_docker

py37_integration:
  stage: integration
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.7 python3.7-dev
    - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli==1.16.272
    - export TOIL_TEST_INTEGRATIVE=True
    - export TOIL_AWS_KEYNAME=id_rsa
    - export TOIL_AWS_ZONE=us-west-2a
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - chmod 400 /root/.ssh/id_rsa
    - make test tests=src/toil/test/jobStores/jobStoreTest.py

py37_provisioner_integration:
  stage: integration
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata awscli jq python3.7 python3.7-dev
    - virtualenv -p python3.7 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor awscli==1.16.272
    - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
    - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
    - export LIBPROCESS_IP=127.0.0.1
    - python setup_gitlab_docker.py
    - export TOIL_TEST_INTEGRATIVE=True; export TOIL_AWS_KEYNAME=id_rsa; export TOIL_AWS_ZONE=us-west-2a
    # This reads GITLAB_SECRET_FILE_SSH_KEYS
    - python setup_gitlab_ssh.py
    - make push_docker
    - make test tests=src/toil/test/sort/sortTest.py
    - make test tests=src/toil/test/provisioners/clusterScalerTest.py
    # - python -m pytest --duration=0 -s -r s src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSRestartTest::testAutoScaledCluster
    # - python -m pytest -s src/toil/test/provisioners/aws/awsProvisionerTest.py
    # - python -m pytest -s src/toil/test/provisioners/gceProvisionerTest.py  # needs env vars set to run

# Python3.8
py38_main:
  stage: basic_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.8 python3.8-dev
    - virtualenv -p python3.8 venv && . venv/bin/activate && make prepare && make develop extras=[all] && pip install htcondor
    - make test tests=src/toil/test/src
    - make test tests=src/toil/test/utils

py38_appliance_build:
  stage: basic_tests
  script:
    - pwd
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata jq python3.8 python3.8-dev
    - virtualenv -p python3.8 venv && . venv/bin/activate && make prepare && pip install pycparser && make develop extras=[all] && pip install htcondor awscli==1.16.272
    # This reads GITLAB_SECRET_FILE_QUAY_CREDENTIALS
    - python setup_gitlab_docker.py
    - make push_docker

# Cactus-on-Kubernetes integration (as a script and not a pytest test)
py37_cactus_integration:
  stage: integration
  script:
    - set -e
    - apt update && DEBIAN_FRONTEND=noninteractive apt install -y tzdata awscli jq python3.7 python3.7-dev
    - virtualenv --system-site-packages --python python3.7 venv
    - . venv/bin/activate
    - pip install .[aws,kubernetes]
    - export TOIL_KUBERNETES_OWNER=toiltest
    - export TOIL_AWS_SECRET_NAME=shared-s3-credentials
    - export TOIL_KUBERNETES_HOST_PATH=/data/scratch
    - export TOIL_WORKDIR=/var/lib/toil
    - export SINGULARITY_CACHEDIR=/var/lib/toil/singularity-cache
    - mkdir -p ${TOIL_WORKDIR}
    - BUCKET_NAME=toil-test-$RANDOM-$RANDOM-$RANDOM
    - cd
    - git clone https://github.com/ComparativeGenomicsToolkit/cactus.git --recursive
    - cd cactus
    - git fetch origin
    - git checkout 12de39de78e784bff1cff5bc022c16e95ded692b
    - git submodule update --init --recursive
    - pip install --upgrade setuptools pip
    - pip install --upgrade .
    - toil clean aws:us-west-2:${BUCKET_NAME}
    - time cactus --batchSystem kubernetes --binariesMode singularity --clean always aws:us-west-2:${BUCKET_NAME} examples/evolverMammals.txt examples/evolverMammals.hal --root mr --defaultDisk "8G" --logDebug --disableCaching false