-
Notifications
You must be signed in to change notification settings - Fork 53
167 lines (151 loc) · 6.27 KB
/
ci.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
name: CI
on:
pull_request:
push:
branches:
- '*'
tags:
- v*
schedule:
# daily (`@daily` not supported, see
# https://docs.github.com/en/actions/reference/events-that-trigger-workflows#scheduled-events )
#
# Runs on default/base branch (see
# https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#onschedule )
- cron: '0 0 * * *'
jobs:
build:
strategy:
fail-fast: false
matrix:
numfocus_nightly: [false]
os: ["ubuntu-latest"]
pyarrow: ["0.17.1", "1.0.1", "2.0.0", "3.0.0", "4.0.1", "5.0.0", "6.0.1", "nightly"]
python: ["3.7", "3.8"]
include:
- numfocus_nightly: true
os: "ubuntu-latest"
pyarrow: "2.0.0"
python: "3.8"
- numfocus_nightly: false
os: "macos-latest"
pyarrow: "0.17.1"
python: "3.8"
continue-on-error: ${{ matrix.numfocus_nightly || matrix.pyarrow == 'nightly' }}
runs-on: ${{ matrix.os }}
defaults:
run:
# see https://github.com/conda-incubator/setup-miniconda/#important
shell: bash -l {0}
env:
IS_MASTER_BUILD: ${{ !matrix.numfocus_nightly && matrix.os == 'ubuntu-latest' && matrix.pyarrow == '0.17.1' && matrix.python == '3.8' }}
IS_TAG: ${{ github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')}}
steps:
# CI setup
- name: Is Master Build?
run: echo $IS_MASTER_BUILD
- name: Checkout
uses: actions/checkout@v2
- name: Cache conda
uses: actions/[email protected]
with:
path: ~/conda_pkgs_dir
key: ${{ runner.os }}-${{ matrix.python }}-${{ matrix.pyarrow }}-${{ hashFiles('conda-requirements.txt', 'conda-test-requirements.txt')}}
- name: Conda Bootstrap
uses: conda-incubator/[email protected]
with:
auto-update-conda: true
channel-priority: strict
channels: conda-forge
python-version: ${{ matrix.python }}
- name: Conda Config
run: |
conda config --set always_yes yes
conda config --set changeps1 no
# Installation into the `test` environment
- name: Conda Install Pyarrow (non-nightly)
run: conda install pyarrow==${{ matrix.pyarrow }}
if: matrix.pyarrow != 'nightly'
- name: Conda Install Pyarrow (nightly)
# Install both arrow-cpp and pyarrow to make sure that we have the
# latest nightly of both packages. It is sadly not guaranteed that the
# nightlies and the latest release would otherwise work together.
run: conda install -c arrow-nightlies arrow-cpp pyarrow
if: matrix.pyarrow == 'nightly'
- name: Conda Install conda-requirements.txt
# run this AFTER pyarrow, since this is less restrictive (so conda doesn't spend too much time downgrading)
run: conda install --file=conda-requirements.txt
- name: Fix Pandas-Pyarrow Interaction
# pandas >=1.2.0 requires newer pyarrow versions, so downgrade pandas
run: conda install 'pandas<1.2'
if: matrix.pyarrow == '0.17.1' && !matrix.numfocus_nightly
- name: Pip Install kartothek
run: pip install --no-deps .
- name: Test import
run: |
python -c "import kartothek"
python -c "import kartothek.api"
python -c "import kartothek.api.consistency"
python -c "import kartothek.api.cube"
python -c "import kartothek.api.dataset"
python -c "import kartothek.api.discover"
python -c "import kartothek.api.serialization"
python -c "import kartothek.cli"
python -c "import kartothek.core"
python -c "import kartothek.io"
python -c "import kartothek.io_components"
python -c "import kartothek.serialization"
python -c "import kartothek.utils"
- name: Conda install test requirements
run: conda install
--file=conda-test-requirements.txt
--freeze-installed
- name: Pip Instal NumFOCUS nightly
# NumFOCUS nightly wheels, contains numpy and pandas
run: pip install --pre --upgrade --timeout=60 --extra-index-url https://pypi.anaconda.org/scipy-wheels-nightly/simple pandas numpy
if: matrix.numfocus_nightly
- name: Conda Export
run: conda list --export
# Tests
- name: Pytest
run: pytest --cov --cov-report xml
- name: Running benchmarks
run: |
asv --config ./asv_bench/asv.conf.json machine --machine github --os unknown --arch unknown --cpu unknown --ram unknown
asv --config ./asv_bench/asv.conf.json dev | sed "/failed$/ s/^/##[error]/" | tee benchmarks.log
if grep "failed" benchmarks.log > /dev/null ; then
exit 1
fi
if: env.IS_MASTER_BUILD == 'true'
# Builds
- name: Build Wheel
run: python setup.py sdist bdist_wheel
- name: Codecov
uses: codecov/[email protected]
with:
# NOTE: `token` is not required, because the kartothek repo is public
file: ./coverage.xml
name: pytest-numfocus_nightly_${{ matrix.numfocus_nightly }}-os_${{ matrix.os }}-pyarrow_${{ matrix.pyarrow }}-python_${{ matrix.python }}
# Release
- name: Publish to PyPI
if: env.IS_MASTER_BUILD == 'true' && env.IS_TAG == 'true'
uses: pypa/gh-action-pypi-publish@master
with:
user: __token__
password: ${{ secrets.PYPI_PASSWORD }}
- name: Create Release Notes
if: env.IS_MASTER_BUILD == 'true' && env.IS_TAG == 'true'
run: |
awk 'BEGIN{found=0} {if (match($0, "==============")) {if (found == 1) exit; found=1}; if (found == 1) {print last}; last=$0}' CHANGES.rst > release_notes.rst
pandoc --from=rst --to=markdown -o release_notes.md release_notes.rst
- name: Create GitHub Release
if: env.IS_MASTER_BUILD == 'true' && env.IS_TAG == 'true'
uses: actions/[email protected]
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
tag_name: ${{ github.ref }}
release_name: Kartothek ${{ github.ref }}
body_path: release_notes.md
draft: false
prerelease: false