-
Notifications
You must be signed in to change notification settings - Fork 5
283 lines (274 loc) · 10.9 KB
/
workflows.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
on: [push, pull_request]
name: 'Test action for workflows'
env:
GALAXY_FORK: galaxyproject
GALAXY_BRANCH: master
MAX_CHUNKS: 4
jobs:
setup-pr-workflows:
name: Setup as in PR for workflows
runs-on: ubuntu-latest
outputs:
galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
repository-list: ${{ steps.discover.outputs.repository-list }}
tool-list: ${{ steps.discover.outputs.tool-list }}
chunk-count: ${{ steps.discover.outputs.chunk-count }}
chunk-list: ${{ steps.discover.outputs.chunk-list }}
strategy:
matrix:
python-version: [3.11]
steps:
- name: Print github context properties
run: |
echo 'event: ${{ github.event_name }}'
echo 'sha: ${{ github.sha }}'
echo 'ref: ${{ github.ref }}'
echo 'head_ref: ${{ github.head_ref }}'
echo 'base_ref: ${{ github.base_ref }}'
echo 'event.before: ${{ github.event.before }}'
echo 'event.after: ${{ github.event.after }}'
- name: Determine latest commit in the Galaxy repo
id: get-galaxy-sha
run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
- name: Cache .planemo
uses: actions/cache@v3
id: cache-planemo
with:
path: ~/.planemo
key: planemo_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
# Install the `wheel` package so that when installing other packages which
# are not available as wheels, pip will build a wheel for them, which can be cached.
- name: Install wheel
run: pip install wheel
- uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Introduce a change on a test workflow
run: |
git config --global user.name "Arthur Dent"
git config --global user.email "[email protected]"
echo "" >> test/workflows/example3/tutorial.ga
git commit -m bump test/workflows/example3/tutorial.ga
- name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks
uses: ./
id: discover
with:
workflows: true
create-cache: ${{ steps.cache-pip.outputs.cache-hit != 'true' || steps.cache-planemo.outputs.cache-hit != 'true' }}
galaxy-fork: ${{ env.GALAXY_FORK }}
galaxy-branch: ${{ env.GALAXY_BRANCH }}
max-chunks: ${{ env.MAX_CHUNKS }}
python-version: ${{ matrix.python-version }}
- name: Check commit range
run: if ! grep "\.\." <<<$(echo ${{ steps.discover.outputs.commit-range }}); then echo "wrong commit range"; exit 1; fi
- name: Check content of repository list
run: |
if ! grep -q "example3" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example3 must be in repo list"; exit 1; fi
if grep -q "example4" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example4 must not be in repo list"; exit 1; fi
- name: Check content of tool list
run: if [ -n "${{ steps.discover.outputs.tool-list }}" ]; then echo "tool list must be empty"; exit 1; fi
- name: Check number of chunks
run: if [ "${{ steps.discover.outputs.chunk-count }}" != "1" ]; then echo "wrong chunk-count"; fi
setup-ci-workflows:
name: Setup as in CI for workflows
runs-on: ubuntu-latest
outputs:
galaxy-head-sha: ${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
fork: ${{ steps.get-fork-branch.outputs.fork }}
branch: ${{ steps.get-fork-branch.outputs.branch }}
repository-list: ${{ steps.discover.outputs.repository-list }}
chunk-count: ${{ steps.discover.outputs.chunk-count }}
chunk-list: ${{ steps.discover.outputs.chunk-list }}
strategy:
matrix:
python-version: [3.11]
steps:
- name: Determine latest commit in the Galaxy repo
id: get-galaxy-sha
run: echo "galaxy-head-sha=$(git ls-remote https://github.com/${{ env.GALAXY_FORK }}/galaxy refs/heads/${{ env.GALAXY_BRANCH }} | cut -f1)" >> $GITHUB_OUTPUT
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ steps.get-galaxy-sha.outputs.galaxy-head-sha }}
# Install the `wheel` package so that when installing other packages which
# are not available as wheels, pip will build a wheel for them, which can be cached.
- name: Install wheel
run: pip install wheel
- uses: actions/checkout@v3
with:
fetch-depth: 1
- name: Artificially exclude tools (because ci_find_repos discovers tool and workflow repos)
run: echo 'test/tools/' > .tt_skip
- name: Fake a Planemo run to update cache and determine commit range, repositories, and chunks
uses: ./
id: discover
env:
GITHUB_EVENT_NAME: schedule
with:
workflows: true
create-cache: true
galaxy-fork: ${{ env.GALAXY_FORK }}
galaxy-branch: ${{ env.GALAXY_BRANCH }}
max-chunks: ${{ env.MAX_CHUNKS }}
python-version: ${{ matrix.python-version }}
github-event-name-override: 'schedule'
- name: Check for empty commit range
run: if [ -n "${{ steps.discover.outputs.commit-range }}" ]; then exit 1; fi
# note: ci_find_repos will find all tools and workflows
- name: Check that all tools an workflows are in the repository list
run: |
if grep -q "tool1" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool1 must not be in the repo list"; exit 1; fi
if grep -q "tool2" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "tool2 must not be in the repo list"; exit 1; fi
if ! grep -q "example3" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example3 must be in the repo list"; exit 1; fi
if ! grep -q "example4" <<<$(echo "${{ steps.discover.outputs.repository-list }}"); then echo "example4 must be in the repo list"; exit 1; fi
# note: ci_find_tools will find only tools, i.e. it should be empty here
- name: Check that all tools are in the tool list
run: if [ -n "${{ steps.discover.outputs.tool-list }}" ]; then exit 1; fi
# chunk-count will only consider tools/workflows depending on the workflow input to the action
- name: Check that the number of chunks is two, i.e. the number of tools
run: if [ "${{ steps.discover.outputs.chunk-count }}" != "2" ]; then exit 1; fi
lint:
name: Test linting of workflows
needs: [setup-ci-workflows]
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.11]
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-workflows.outputs.galaxy-head-sha }}
- name: Install wheel
run: pip install wheel
- name: Planemo lint workflows
uses: ./
with:
mode: lint
workflows: true
repository-list: "${{ needs.setup-ci-workflows.outputs.repository-list }}"
tool-list: "${{ needs.setup-ci-workflows.outputs.tool-list }}"
test-workflows:
name: Test testing of workflows
needs: [setup-ci-workflows]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
chunk: ${{ fromJson(needs.setup-ci-workflows.outputs.chunk-list) }}
python-version: [3.11]
services:
postgres:
image: postgres:11
env:
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_DB: postgres
ports:
- 5432:5432
steps:
# checkout the repository
# and use it as the current working directory
- uses: actions/checkout@v3
with:
fetch-depth: 1
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-workflows.outputs.galaxy-head-sha }}
- name: Planemo test workflows
uses: ./
id: test-workflows
with:
mode: test
workflows: true
setup-cvmfs: true
repository-list: ${{ needs.setup-ci-workflows.outputs.repository-list }}
galaxy-fork: ${{ env.GALAXY_FORK }}
galaxy-branch: ${{ env.GALAXY_BRANCH }}
chunk: ${{ matrix.chunk }}
chunk-count: ${{ needs.setup-ci-workflows.outputs.chunk-count }}
- name: 2nd run test workflows
uses: ./
id: test-workflows-2
with:
mode: test
workflows: true
setup-cvmfs: false
repository-list: ${{ needs.setup-ci-workflows.outputs.repository-list }}
galaxy-fork: ${{ env.GALAXY_FORK }}
galaxy-branch: ${{ env.GALAXY_BRANCH }}
chunk: ${{ matrix.chunk }}
chunk-count: ${{ needs.setup-ci-workflows.outputs.chunk-count }}
- uses: actions/upload-artifact@v3
with:
name: 'Workflow test output ${{ matrix.chunk }}'
path: upload
combine_outputs:
name: Test combining 'chunked' test results
needs: [setup-ci-workflows, test-workflows]
strategy:
matrix:
python-version: [3.11]
# This job runs on Linux
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 1
- uses: actions/download-artifact@v3
with:
path: artifacts
- uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache .cache/pip
uses: actions/cache@v3
id: cache-pip
with:
path: ~/.cache/pip
key: pip_cache_py_${{ matrix.python-version }}_gxy_${{ needs.setup-ci-tools.outputs.galaxy-head-sha }}
- name: Combine outputs
uses: ./
id: combine
with:
mode: combine
html-report: true
- name: Check statistics
run: if ! grep -q "4\s\+success" <<<$(echo ${{ steps.combine.outputs.statistics }}); then echo "wrong statistics"; exit 1; fi
- uses: actions/upload-artifact@v3
with:
name: 'All tool test results'
path: upload
- name: Check outputs
uses: ./
id: check
with:
mode: check