forked from broadinstitute/gatk
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy path.travis.yml
208 lines (202 loc) · 9.36 KB
/
.travis.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
language: java
sudo: required
dist: trusty
group: travis_lts
git:
depth: 9999999
lfs_skip_smudge: true
jdk:
- openjdk8
env:
matrix:
- TEST_TYPE=cloud UPLOAD=true TESTS_REQUIRE_GCLOUD=true
- TEST_TYPE=integration TEST_DOCKER=true TEST_VERBOSITY=minimal
- TEST_TYPE=unit TEST_DOCKER=true TEST_VERBOSITY=minimal
- TEST_TYPE=variantcalling TEST_DOCKER=true TEST_VERBOSITY=minimal
- TEST_TYPE=python TEST_DOCKER=true TEST_VERBOSITY=minimal
- RUN_CNV_GERMLINE_COHORT_WDL=true TESTS_REQUIRE_GCLOUD=true
- RUN_CNV_GERMLINE_CASE_WDL=true TESTS_REQUIRE_GCLOUD=true
- RUN_CNV_SOMATIC_WDL=true TESTS_REQUIRE_GCLOUD=true
- RUN_M2_WDL=true TESTS_REQUIRE_GCLOUD=true
- RUN_CNN_WDL=true TESTS_REQUIRE_GCLOUD=true
global:
#gradle needs this
- TERM=dumb
#limit gradle jvm memory and disable daemon
- GRADLE_OPTS="-Xmx2048m -Dorg.gradle.daemon=false"
#google cloud stuff
- export CLOUDSDK_CORE_DISABLE_PROMPTS=1
- export GCLOUD_HOME=$HOME/gcloud/google-cloud-sdk/bin
- export CLOUDSDK_PYTHON_SITEPACKAGES=1
- export GOOGLE_APPLICATION_CREDENTIALS=$TRAVIS_BUILD_DIR/servicekey.json
#hellbender specific variables
- HELLBENDER_TEST_INPUTS=gs://hellbender/test/resources/
- HELLBENDER_TEST_STAGING=gs://hellbender-test-logs/staging/
- HELLBENDER_TEST_LOGS=/hellbender-test-logs/build_reports/
- HELLBENDER_TEST_PROJECT=broad-dsde-dev
- HELLBENDER_JSON_SERVICE_ACCOUNT_KEY=servicekey.json
#for uploading artifacts
- ARTIFACTORY_USERNAME=gatkci
#artifactory password
- secure: "E0LWXgX3aWSE/DWHXXDx4vrAq4uX6vKg402wToaZ5otbHQ/UP0H7/FA5jomavAXoC46oMVHZcEltZ5OVhuJ0NW8yYxUCecJ1D/YvVQmnfFABcV/qLM+k4e2rYQOKVw/pejB2gG8XdTA+XE2WyTeENbmIkputS8f1ndKWCmZxuuk="
# For cromwell jar download
- CROMWELL_JAR=$HOME/cromwell-36.jar
matrix:
fast_finish: true
include:
- jdk: oraclejdk8
env: TEST_TYPE=integration TEST_VERBOSITY=minimal TEST_REQUIRE_GCLOUD=true
before_cache:
- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock
- rm -fr $HOME/.gradle/caches/*/plugin-resolution/
cache:
directories:
- $HOME/.gradle/caches/
- $HOME/.gradle/wrapper/
- $HOME/.m2
- $HOME/gcloud/
- $HOME/site-library
before_install:
#skip push builds if there's an associated PR because we don't look at them anway
- if [[ ${TRAVIS_EVENT_TYPE} == push ]];
then
PULL_REQUESTS=$( scripts/travis/check_for_pull_request );
if [[ $( grep -c "commits" <<< ${PULL_REQUESTS} ) -gt 0 ]];
then
travis_terminate 0;
fi;
fi;
- REPORT_PATH=${TRAVIS_BRANCH}_${TRAVIS_JOB_NUMBER};
- if [[ $TRAVIS_SECURE_ENV_VARS == true && $TRAVIS_EVENT_TYPE != cron ]]; then
echo "Test report will be written to https://storage.googleapis.com$HELLBENDER_TEST_LOGS$REPORT_PATH/tests/test/index.html";
fi;
- if [[ ${TRAVIS_SECURE_ENV_VARS} == true ]]; then
if [[ ${TESTS_REQUIRE_GCLOUD} == true ]]; then
scripts/travis/install_and_authenticate_to_gcloud.sh;
fi;
openssl aes-256-cbc -K $encrypted_4823d58debd3_key -iv $encrypted_4823d58debd3_iv -in resources_for_CI/github_deploy_key.enc -out ~/.ssh/id_rsa -d;
chmod 600 ~/.ssh/id_rsa;
echo "Testing github authentication:";
ssh -T [email protected];
echo "Done testing github authentication";
fi;
#install R
- if [[ $TEST_DOCKER != true ]]; then
sudo mkdir -p /usr/local/lib/R/;
sudo mkdir -p site-library;
sudo ln -sFv ~/site-library /usr/local/lib/R/site-library;
sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9;
sudo add-apt-repository "deb http://cran.rstudio.com/bin/linux/ubuntu trusty/";
sudo apt-get update;
sudo apt-get install -y --force-yes r-base-dev=3.1.3-1trusty;
sudo apt-get install -y --force-yes r-base-core=3.1.3-1trusty;
sudo Rscript scripts/docker/gatkbase/install_R_packages.R;
fi;
# Download Cromwell jar -- if you change the version, please change the CROMWELL_JAR env variable above, too.
- if [[ $RUN_CNV_GERMLINE_COHORT_WDL == true || $RUN_CNV_GERMLINE_CASE_WDL == true || $RUN_CNV_SOMATIC_WDL == true || $RUN_M2_WDL == true || $RUN_CNN_WDL == true ]]; then
wget -O $CROMWELL_JAR https://github.com/broadinstitute/cromwell/releases/download/36/cromwell-36.jar;
fi;
# Download git lfs files
- git lfs version
- git lfs install
- git lfs fetch
- git lfs checkout
install:
- if [[ $TRAVIS_SECURE_ENV_VARS == false && $TEST_TYPE == cloud ]]; then
echo "Can't run cloud tests without keys so don't bother building";
elif [[ $TEST_DOCKER == true ]]; then
echo "Skip the install because we're doing a docker build";
else
./gradlew assemble;
./gradlew installDist;
if [[ $RUN_CNV_GERMLINE_COHORT_WDL == true || $RUN_CNV_GERMLINE_CASE_WDL == true || $RUN_CNV_SOMATIC_WDL == true || $RUN_M2_WDL == true || $RUN_CNN_WDL == true ]]; then
echo "building a shadow jar for the wdl";
./gradlew shadowJar;
elif [[ $TEST_TYPE == cloud ]]; then
echo "building a spark jar for the dataproc tests";
./gradlew sparkjar;
fi;
fi;
script:
# run a basic sanity check to be sure that gatk doesn't explode
# run tests
- if [[ $TRAVIS_EVENT_TYPE == cron ]]; then
echo "Not running any tests for nightly builds";
elif [[ $TRAVIS_SECURE_ENV_VARS == false && $TEST_TYPE == cloud ]]; then
echo "Can't run cloud tests without keys so don't run tests";
elif [[ $RUN_CNV_GERMLINE_COHORT_WDL == true ]]; then
echo "Running CNV germline cohort workflow";
travis_wait 60 bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh COHORT;
elif [[ $RUN_CNV_GERMLINE_CASE_WDL == true ]]; then
echo "Running CNV germline case workflow";
travis_wait 60 bash scripts/cnv_cromwell_tests/germline/run_cnv_germline_workflows.sh CASE;
elif [[ $RUN_CNV_SOMATIC_WDL == true ]]; then
echo "Running CNV somatic workflows";
bash scripts/cnv_cromwell_tests/somatic/run_cnv_somatic_workflows.sh;
elif [[ $RUN_M2_WDL == true ]]; then
echo "Deleting some unused files before running M2 WDL...";
rm -Rf src/test/resources/large/VQSR;
echo "Running M2 WDL";
sudo bash scripts/m2_cromwell_tests/run_m2_wdl.sh;
elif [[ $RUN_CNN_WDL == true ]]; then
echo "Running CNN WDL";
sudo bash scripts/cnn_variant_cromwell_tests/run_cnn_variant_wdl.sh;
elif [[ $TEST_DOCKER == true ]]; then
echo "Building docker image and running appropriate tests..." ;
if [ ${TRAVIS_PULL_REQUEST} != false ]; then
sudo bash build_docker.sh -e FETCH_HEAD -s -u -t ${TRAVIS_PULL_REQUEST};
DOCKER_TAG=FETCH_HEAD;
else
echo ${TRAVIS_COMMIT};
sudo bash build_docker.sh -e ${TRAVIS_COMMIT} -s -u;
DOCKER_TAG=$TRAVIS_COMMIT;
fi;
sudo docker images;
echo ${TEST_TYPE};
sudo mkdir -p build/reports/;
sudo chmod -R a+w build/reports/;
cp scripts/docker/dockertest.gradle .;
sudo docker run -v $(pwd):/gatkCloneMountPoint:cached -v $(pwd)/testJars:/jars:cached --rm -e "TEST_VERBOSITY=minimal" -e "TEST_TYPE=${TEST_TYPE}" -t broadinstitute/gatk:${DOCKER_TAG} bash --init-file /gatk/gatkenv.rc /root/run_unit_tests.sh;
TEST_EXIT_VALUE=$?;
sudo mkdir build/reports/tests/test && sudo cp -rp build/reports/tests/testOnPackagedReleaseJar/* build/reports/tests/test && sudo rm -r build/reports/tests/testOnPackagedReleaseJar;
$( exit ${TEST_EXIT_VALUE} );
else
./gatk PrintReads -I src/test/resources/NA12878.chr17_69k_70k.dictFix.bam -O output.bam;
travis_wait 50 ./gradlew jacocoTestReport;
fi;
# This creates and uploads the gatk zip file to the nightly build bucket, only keeping the 10 newest entries
# This also constructs the Docker image and uploads it to https://cloud.docker.com/u/broadinstitute/repository/docker/broadinstitute/gatk-nightly/
- if [[ $TRAVIS_BRANCH == master && $TRAVIS_EVENT_TYPE == cron && $UPLOAD == true ]]; then
docker login --username=$DOCKER_SERVICE_LOGIN --password=$DOCKER_SERVICE_PASS;
$GCLOUD_HOME/gcloud components -q update gsutil;
gsutil ls -l gs://gatk-nightly-builds | grep gatk | sort -r -k 2 | grep -o '\S\+$' | tail -n +11 | xargs -I {} gsutil rm {};
./gradlew bundle;
ZIP_FILE="$(ls build/ | grep '^gatk.*.zip$' | grep -iv python)";
echo "Uploading zip to gs://gatk-nightly-builds/";
$GCLOUD_HOME/gsutil -m cp build/$ZIP_FILE gs://gatk-nightly-builds/"$(date +%Y-%m-%d)"-$ZIP_FILE;
bash build_docker.sh -e ${TRAVIS_COMMIT} -s -u;
DOCKER_TAG=$(docker images -q | head -n 1);
DATE=$(date +%Y-%m-%d);
DOCKER_NIGHTLY_TAG=broadinstitute/gatk-nightly:$DATE-$(git describe)-NIGHTLY-SNAPSHOT;
echo "Pushing Nightly Docker image";
docker tag $DOCKER_TAG $DOCKER_NIGHTLY_TAG;
docker push $DOCKER_NIGHTLY_TAG;
fi;
after_success:
- bash <(curl -s https://codecov.io/bash)
- if [[ $TRAVIS_BRANCH == master && $UPLOAD == true ]]; then ./gradlew uploadArchives; fi;
# if this is actually a commit to master and not a pull request build into master, then publish master-snapshot
- if [[ $TRAVIS_BRANCH == master && $TRAVIS_PULL_REQUEST == false && $UPLOAD == true ]]; then
git tag master;
./gradlew uploadArchives;
fi;
after_failure:
- dmesg | tail -100
after_script:
- if [[ $TRAVIS_SECURE_ENV_VARS == true && $TRAVIS_EVENT_TYPE != cron ]]; then
if [[ ${TESTS_REQUIRE_GCLOUD} != "true" ]]; then
scripts/travis/install_and_authenticate_to_gcloud.sh;
fi;
$GCLOUD_HOME/gsutil -m cp -z html -z js -z xml -z css -r build/reports/tests gs:/$HELLBENDER_TEST_LOGS$REPORT_PATH/;
echo "See the test report at https://storage.googleapis.com$HELLBENDER_TEST_LOGS$REPORT_PATH/tests/test/index.html";
fi;