-
Notifications
You must be signed in to change notification settings - Fork 10
236 lines (216 loc) · 8.83 KB
/
build-and-test.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
name: xmos-ai-tools-build-and-test
on:
workflow_dispatch:
inputs:
version:
description: "Provide a valid branch or commit hash"
required: true
pull_request:
# Save the tag version in an environment variable
# The pretend version is used for the wheel as the
# tag version might be a branch name or commit hash
env:
TAG_VERSION: ${{ github.event.inputs.version }}
PRETEND_VERSION: 0.1.1
BAZEL_CACHE_URL: https://storage.googleapis.com/bazel_remote_cache_0/gh_actions
jobs:
macos-release-wheel:
name: Build and test release wheels for macOS
runs-on: macos-latest
strategy:
matrix:
python-version: [3.9]
fail-fast: false
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
ref: ${{ env.TAG_VERSION }}
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- uses: google-github-actions/auth@v0
continue-on-error: true
with:
create_credentials_file: true
credentials_json: ${{ secrets.BAZEL_REMOTE_CACHE_JSON_KEY }}
- name: Build setup
env:
SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.PRETEND_VERSION }}
shell: bash
run: |
python --version
python -m pip install wheel setuptools setuptools-scm numpy six
cd $GITHUB_WORKSPACE/third_party/lib_tflite_micro
make patch
cd $GITHUB_WORKSPACE/python/xmos_ai_tools/xinterpreters
mkdir -p build
cd build
cmake ..
cmake --build . -t install --parallel 3 --config Release
- name: Build xformer
shell: bash
run: |
cd $GITHUB_WORKSPACE/xformer
if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
echo "using Google Cloud Storage cache"
echo -e 'build --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }}' >> .bazelrc.user
echo -e 'build --remote_upload_local_results=true --experimental_remote_cache_compression' >> .bazelrc.user
echo -e 'build --google_default_credentials' >> .bazelrc.user
fi
bazelisk build //:xcore-opt --verbose_failures --copt=-fvisibility=hidden --copt=-mavx --copt=-mmacosx-version-min=10.13 --linkopt=-mmacosx-version-min=10.13 --linkopt=-dead_strip --distinct_host_configuration=false --//:disable_version_check
- name: Build macOS wheels
env:
SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.PRETEND_VERSION }}
shell: bash
run: |
cd $GITHUB_WORKSPACE/python
python setup.py bdist_wheel --plat macosx_10_9_x86_64
- name: Test macOS wheels
run: |
pip install -r ./requirements.txt
cd python/dist
pip install *
cd ../..
pytest integration_tests/runner.py --models_path integration_tests/models/non-bnns -n 3 --reruns 5 --reruns-delay 1
manylinux-release-wheel:
name: Build and test release wheels for manylinux2014
runs-on: ubuntu-latest
strategy:
matrix:
python-version: [3.9]
fail-fast: false
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
ref: ${{ env.TAG_VERSION }}
- name: Free Disk Space (Ubuntu)
uses: jlumbroso/free-disk-space@main
with:
# this might remove tools that are actually needed,
# if set to "true" but frees about 6 GB
tool-cache: false
# all of these default to true, but feel free to set to
# "false" if necessary for your workflow
android: true
dotnet: true
haskell: true
large-packages: false
docker-images: false
swap-storage: false
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- uses: google-github-actions/auth@v0
continue-on-error: true
with:
create_credentials_file: true
credentials_json: ${{ secrets.BAZEL_REMOTE_CACHE_JSON_KEY }}
- name: Build xformer and manylinux2014 wheel in docker
# The SETUPTOOLS_SCM_PRETEND_VERSION var is passed via docker to the
# python setup.py script to add the version info for the wheel file.
#
# auditwheel requires patchelf and repairs the created wheel file to
# be compatible for manylinux.
run: |
pushd $GITHUB_WORKSPACE/xformer
if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
echo "using Google Cloud Storage cache"
echo -e 'build --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }}' >> .bazelrc.user
echo -e 'build --remote_upload_local_results=true --experimental_remote_cache_compression' >> .bazelrc.user
echo -e 'build --google_default_credentials' >> .bazelrc.user
fi
popd
cd $GITHUB_WORKSPACE
docker run -e SETUPTOOLS_SCM_PRETEND_VERSION=${{ env.PRETEND_VERSION }} \
-e GOOGLE_APPLICATION_CREDENTIALS=/tmp/gcloud-credentials.json \
-v $GOOGLE_APPLICATION_CREDENTIALS:/tmp/gcloud-credentials.json:ro \
-v ${PWD}:/ai_tools -w /ai_tools \
tensorflow/build:2.15-python${{ matrix.python-version }} \
.github/scripts/build_linux_wheel.sh
sudo apt-get -y -qq install patchelf --no-install-recommends
python -m pip install auditwheel==5.2.0
cd python
for f in dist/*.whl; do
ls -l $f
auditwheel repair --plat manylinux2014_x86_64 $f
ls -l $f
done
- name: Test linux wheels
run: |
pip install -r ./requirements.txt
cd python/wheelhouse
pip install *
cd ../..
pytest integration_tests/runner.py --models_path integration_tests/models/non-bnns -n 2 --reruns 5 --reruns-delay 1
windows-release-wheel:
name: Build and test release wheels for windows
runs-on: windows-2019
strategy:
matrix:
python-version: [3.9]
fail-fast: false
steps:
- uses: actions/checkout@v2
with:
submodules: recursive
fetch-depth: 0
ref: ${{ env.TAG_VERSION }}
- uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
cache: 'pip'
- uses: google-github-actions/auth@v0
continue-on-error: true
with:
create_credentials_file: true
credentials_json: ${{ secrets.BAZEL_REMOTE_CACHE_JSON_KEY }}
- name: Configure pagefile
uses: al-cheb/[email protected]
with:
minimum-size: 10GB
maximum-size: 20GB
- name: Set up pagefile
shell: pwsh
run: |
(Get-CimInstance Win32_PageFileUsage).AllocatedBaseSize
- name: Build setup
shell: cmd
run: |
python --version
python -m pip install wheel setuptools setuptools-scm numpy six
python -c "import sys; print(sys.executable)"
call "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvars64.bat"
cd %GITHUB_WORKSPACE%/third_party/lib_tflite_micro
make patch
cd %GITHUB_WORKSPACE%/python/xmos_ai_tools/xinterpreters
mkdir -p build
cd build
cmake ..
cmake --build . -t install --parallel 2 --config Release
- name: Build xformer
shell: cmd
run: |
cd %GITHUB_WORKSPACE%/xformer
IF DEFINED GOOGLE_APPLICATION_CREDENTIALS set "BAZEL_EXTRA_ARGS=--remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }} --google_default_credentials --remote_upload_local_results=true --experimental_remote_cache_compression"
set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC
bazelisk --output_user_root c:\_bzl build //:xcore-opt --verbose_failures --local_ram_resources=5120 --action_env PYTHON_BIN_PATH="C:/hostedtoolcache/windows/Python/3.9.13/x64/python.exe" %BAZEL_EXTRA_ARGS% --//:disable_version_check
- name: Build windows wheels
env:
SETUPTOOLS_SCM_PRETEND_VERSION: ${{ env.PRETEND_VERSION }}
shell: cmd
run: |
cd python
python setup.py bdist_wheel
- name: Test windows wheels
run: |
pip install -r ./requirements.txt
cd python/dist
pip install xmos_ai_tools-0.1.1-py3-none-win_amd64.whl
cd ../..
pytest integration_tests/runner.py --models_path integration_tests/models/non-bnns -n 2 --reruns 5 --reruns-delay 1