From 20b2cb1fb60d0df90321db45445898cd86276699 Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Tue, 19 Sep 2023 09:52:06 +0100 Subject: [PATCH 01/16] Add build to readthedocs config --- .readthedocs.yaml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.readthedocs.yaml b/.readthedocs.yaml index aa61218..326f882 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -1,6 +1,9 @@ version: 2 formats: [] +build: + os: "ubuntu-22.04" + tools: + python: "3.9" python: - version: 3.8 install: - - requirements: docs/requirements.txt + - requirements: docs/requirements.txt From b910e746d1f45aef9732f79c7309725adf5c68d2 Mon Sep 17 00:00:00 2001 From: Iain Russell Date: Wed, 18 Oct 2023 17:52:58 +0100 Subject: [PATCH 02/16] Fix test that fails with pandas 2.1.1 (#71) --- tests/test_40_sample_data.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/test_40_sample_data.py b/tests/test_40_sample_data.py index fd1d7c3..8dffeba 100644 --- a/tests/test_40_sample_data.py +++ b/tests/test_40_sample_data.py @@ -745,7 +745,7 @@ def test_sat_compressed_1() -> None: ] expected_first_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:05"), + "data_datetime": pd.Timestamp("2015-08-21 01:59:05").as_unit("ns"), "latitude": -44.83389, "longitude": 171.1635, "nonCoordinateLatitude": -44.82399, @@ -755,7 +755,7 @@ def test_sat_compressed_1() -> None: } expected_second_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:05"), + "data_datetime": pd.Timestamp("2015-08-21 01:59:05").as_unit("ns"), "latitude": -44.83389, "longitude": 171.1635, "nonCoordinateLatitude": -44.82399, @@ -765,7 +765,7 @@ def test_sat_compressed_1() -> None: } expected_12_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:05"), + "data_datetime": pd.Timestamp("2015-08-21 01:59:05").as_unit("ns"), "latitude": -44.83389, "longitude": 171.1635, "nonCoordinateLatitude": -44.82399, @@ -775,7 +775,7 @@ def test_sat_compressed_1() -> None: } expected_13_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:06"), + "data_datetime": pd.Timestamp("2015-08-21 01:59:06").as_unit("ns"), "latitude": -44.77121, "longitude": 171.1515, "nonCoordinateLatitude": -44.76132, From dcb4a2fb3816de87b10d4bd9f3d9da5f631a0cff Mon Sep 17 00:00:00 2001 From: Iain Russell Date: Wed, 18 Oct 2023 18:09:30 +0100 Subject: [PATCH 03/16] Fix test that fails with pandas 1.5.3 (#71) --- tests/test_40_sample_data.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tests/test_40_sample_data.py b/tests/test_40_sample_data.py index 8dffeba..f88d011 100644 --- a/tests/test_40_sample_data.py +++ b/tests/test_40_sample_data.py @@ -734,6 +734,15 @@ def test_ens_compressed() -> None: def test_sat_compressed_1() -> None: + + def timestamp(s): + ts = pd.Timestamp(s) + try: + ts = ts.as_unit("ns") + except: + pass + return ts + columns = [ "data_datetime", "latitude", @@ -745,7 +754,7 @@ def test_sat_compressed_1() -> None: ] expected_first_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:05").as_unit("ns"), + "data_datetime": timestamp("2015-08-21 01:59:05"), "latitude": -44.83389, "longitude": 171.1635, "nonCoordinateLatitude": -44.82399, @@ -755,7 +764,7 @@ def test_sat_compressed_1() -> None: } expected_second_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:05").as_unit("ns"), + "data_datetime": timestamp("2015-08-21 01:59:05"), "latitude": -44.83389, "longitude": 171.1635, "nonCoordinateLatitude": -44.82399, @@ -765,7 +774,7 @@ def test_sat_compressed_1() -> None: } expected_12_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:05").as_unit("ns"), + "data_datetime": timestamp("2015-08-21 01:59:05"), "latitude": -44.83389, "longitude": 171.1635, "nonCoordinateLatitude": -44.82399, @@ -775,7 +784,7 @@ def test_sat_compressed_1() -> None: } expected_13_row = { - "data_datetime": pd.Timestamp("2015-08-21 01:59:06").as_unit("ns"), + "data_datetime": timestamp("2015-08-21 01:59:06"), "latitude": -44.77121, "longitude": 171.1515, "nonCoordinateLatitude": -44.76132, From 205d71b6b3cae0fbde3d575574bc76e71d75c67b Mon Sep 17 00:00:00 2001 From: Iain Russell Date: Wed, 18 Oct 2023 19:36:53 +0100 Subject: [PATCH 04/16] Mypy (#71) --- tests/test_40_sample_data.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_40_sample_data.py b/tests/test_40_sample_data.py index f88d011..a0d2a58 100644 --- a/tests/test_40_sample_data.py +++ b/tests/test_40_sample_data.py @@ -735,7 +735,7 @@ def test_ens_compressed() -> None: def test_sat_compressed_1() -> None: - def timestamp(s): + def timestamp(s: str) -> T.Any: ts = pd.Timestamp(s) try: ts = ts.as_unit("ns") From 09f8583d418c66ad75e6a155cd12288e40385d19 Mon Sep 17 00:00:00 2001 From: Iain Russell Date: Wed, 18 Oct 2023 19:38:54 +0100 Subject: [PATCH 05/16] Black (#71) --- tests/test_40_sample_data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/test_40_sample_data.py b/tests/test_40_sample_data.py index a0d2a58..251ba2d 100644 --- a/tests/test_40_sample_data.py +++ b/tests/test_40_sample_data.py @@ -734,7 +734,6 @@ def test_ens_compressed() -> None: def test_sat_compressed_1() -> None: - def timestamp(s: str) -> T.Any: ts = pd.Timestamp(s) try: From 33558482dda2923c7416192a9ccd92b0135259aa Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Thu, 24 Aug 2023 15:23:55 +0100 Subject: [PATCH 06/16] Add downstream ci --- .github/ci-config.yml | 7 ++++ .github/ci-hpc-config.yml | 8 +++++ .github/workflows/ci.yml | 76 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 .github/ci-config.yml create mode 100644 .github/ci-hpc-config.yml create mode 100644 .github/workflows/ci.yml diff --git a/.github/ci-config.yml b/.github/ci-config.yml new file mode 100644 index 0000000..e5291d2 --- /dev/null +++ b/.github/ci-config.yml @@ -0,0 +1,7 @@ +dependencies: | + ecmwf/ecbuild + MathisRosenhauer/libaec@master + ecmwf/eccodes + ecmwf/odc +dependency_branch: develop +parallelism_factor: 8 \ No newline at end of file diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml new file mode 100644 index 0000000..f872dc4 --- /dev/null +++ b/.github/ci-hpc-config.yml @@ -0,0 +1,8 @@ +build: + modules: + - ninja + dependencies: + - ecmwf/ecbuild@develop + - ecmwf/eccodes@develop + - ecmwf/odc@develop + parallel: 64 \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..ddda8fc --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,76 @@ +name: ci + +on: + # Trigger the workflow on push to master or develop, except tag creation + push: + branches: + - 'master' + - 'develop' + tags-ignore: + - '**' + + # Trigger the workflow on pull request + pull_request: ~ + + # Trigger the workflow manually + workflow_dispatch: ~ + + # Trigger after public PR approved for CI + pull_request_target: + types: [labeled] + +jobs: + # Run CI including downstream packages on self-hosted runners + downstream-ci: + name: downstream-ci + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main + with: + metkit: pdbufr/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} + codecov_upload: true + secrets: inherit + + # # Run CI of private downstream packages on self-hosted runners + # private-downstream-ci: + # name: private-downstream-ci + # needs: [downstream-ci] + # if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + # runs-on: ubuntu-latest + # permissions: + # pull-requests: write + # steps: + # - name: Dispatch private downstream CI + # uses: ecmwf-actions/dispatch-private-downstream-ci@v1 + # with: + # token: ${{ secrets.GH_REPO_READ_TOKEN }} + # owner: ecmwf-actions + # repository: private-downstream-ci + # event_type: downstream-ci + # payload: '{"metkit": "ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }}"}' + + # Build downstream packages on HPC + downstream-ci-hpc: + name: downstream-ci-hpc + if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main + with: + pdbufr: ecmwf/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} + secrets: inherit + + # # Run CI of private downstream packages on HPC + # private-downstream-ci-hpc: + # name: private-downstream-ci-hpc + # needs: [downstream-ci-hpc] + # if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} + # runs-on: ubuntu-latest + # permissions: + # pull-requests: write + # steps: + # - name: Dispatch private downstream CI + # uses: ecmwf-actions/dispatch-private-downstream-ci@v1 + # with: + # token: ${{ secrets.GH_REPO_READ_TOKEN }} + # owner: ecmwf-actions + # repository: private-downstream-ci + # event_type: downstream-ci-hpc + # payload: '{"metkit": "ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }}"}' \ No newline at end of file From 823b8249accc092bbf2c2f796a02a89fd4b44d55 Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Thu, 24 Aug 2023 15:28:07 +0100 Subject: [PATCH 07/16] Add downstream ci --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ddda8fc..779a9c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@feature/add-pdbufr with: metkit: pdbufr/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true @@ -52,7 +52,7 @@ jobs: downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/add-pdbufr with: pdbufr: ecmwf/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit From 0ce97d1a413c76db2c39d460eccd9cd5ef5eb5fb Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Thu, 24 Aug 2023 15:40:54 +0100 Subject: [PATCH 08/16] Add downstream ci --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 779a9c8..c8f8078 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@feature/add-pdbufr + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@feature/add-pfbufr with: metkit: pdbufr/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true @@ -52,7 +52,7 @@ jobs: downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/add-pdbufr + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/add-pfbufr with: pdbufr: ecmwf/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit From 1a9b44fd0423ae21e9c3d914fb09ccd0c39023c1 Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Thu, 24 Aug 2023 15:48:25 +0100 Subject: [PATCH 09/16] Add downstream ci --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8f8078..c3a9839 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,7 @@ jobs: if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@feature/add-pfbufr with: - metkit: pdbufr/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} + pdbufr: ecmwf/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true secrets: inherit From 05055b8d964df00162c6f3ef2b453e418ff07a20 Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Thu, 24 Aug 2023 15:56:56 +0100 Subject: [PATCH 10/16] Add downstream ci --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 7f96c44..de5bee7 100644 --- a/setup.py +++ b/setup.py @@ -40,7 +40,7 @@ def parse_version_from(path: str) -> str: url="https://github.com/ecmwf/pdbufr", packages=setuptools.find_packages(), include_package_data=True, - install_requires=["attrs", "eccodes", "pandas"], + install_requires=["attrs", "eccodes", "pandas", "hypothesis"], extras_require={"tests": ["flake8", "pytest", "pytest-cov", "requests"]}, zip_safe=True, keywords="eccodes bufr pandas", From 8ee7456395cc76e29f851bd5ea0412c05af7b33d Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Thu, 24 Aug 2023 16:29:12 +0100 Subject: [PATCH 11/16] Add downstream ci --- .github/ci-config.yml | 1 - .github/ci-hpc-config.yml | 1 - 2 files changed, 2 deletions(-) diff --git a/.github/ci-config.yml b/.github/ci-config.yml index e5291d2..15d1217 100644 --- a/.github/ci-config.yml +++ b/.github/ci-config.yml @@ -2,6 +2,5 @@ dependencies: | ecmwf/ecbuild MathisRosenhauer/libaec@master ecmwf/eccodes - ecmwf/odc dependency_branch: develop parallelism_factor: 8 \ No newline at end of file diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml index f872dc4..2e44c1a 100644 --- a/.github/ci-hpc-config.yml +++ b/.github/ci-hpc-config.yml @@ -4,5 +4,4 @@ build: dependencies: - ecmwf/ecbuild@develop - ecmwf/eccodes@develop - - ecmwf/odc@develop parallel: 64 \ No newline at end of file From 1f2803da86103b5db5a1e1310cbb45c21d065f8a Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Thu, 24 Aug 2023 17:07:11 +0100 Subject: [PATCH 12/16] Add downstream ci --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3a9839..8588c67 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: downstream-ci: name: downstream-ci if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@feature/add-pfbufr + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main with: pdbufr: ecmwf/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} codecov_upload: true @@ -52,7 +52,7 @@ jobs: downstream-ci-hpc: name: downstream-ci-hpc if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }} - uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@feature/add-pfbufr + uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main with: pdbufr: ecmwf/pdbufr@${{ github.event.pull_request.head.sha || github.sha }} secrets: inherit From 5fea7ed98e3e8570d45adfa2b43e11f585e000ae Mon Sep 17 00:00:00 2001 From: Dusan Figala Date: Thu, 31 Aug 2023 12:48:42 +0200 Subject: [PATCH 13/16] Amend CI config for python --- .github/ci-config.yml | 3 ++- .github/ci-hpc-config.yml | 8 +++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/ci-config.yml b/.github/ci-config.yml index 15d1217..77c9e81 100644 --- a/.github/ci-config.yml +++ b/.github/ci-config.yml @@ -3,4 +3,5 @@ dependencies: | MathisRosenhauer/libaec@master ecmwf/eccodes dependency_branch: develop -parallelism_factor: 8 \ No newline at end of file +parallelism_factor: 8 +self_build: false diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml index 2e44c1a..f2516d5 100644 --- a/.github/ci-hpc-config.yml +++ b/.github/ci-hpc-config.yml @@ -1,7 +1,13 @@ build: + python: '3.10' modules: - ninja dependencies: - ecmwf/ecbuild@develop - ecmwf/eccodes@develop - parallel: 64 \ No newline at end of file + python_dependencies: + - ecmwf/eccodes-python@develop + env: + - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples + - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions + parallel: 64 From 22d3c40adc4fb0eae46a99132f71112716e6aaf1 Mon Sep 17 00:00:00 2001 From: Dusan Figala Date: Tue, 10 Oct 2023 15:04:37 +0200 Subject: [PATCH 14/16] Add downstream CI requirements --- .github/ci-hpc-config.yml | 1 + tests/downstream-ci-requirements.txt | 1 + 2 files changed, 2 insertions(+) create mode 100644 tests/downstream-ci-requirements.txt diff --git a/.github/ci-hpc-config.yml b/.github/ci-hpc-config.yml index f2516d5..5996da1 100644 --- a/.github/ci-hpc-config.yml +++ b/.github/ci-hpc-config.yml @@ -11,3 +11,4 @@ build: - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions parallel: 64 + requirements: tests/downstream-ci-requirements.txt diff --git a/tests/downstream-ci-requirements.txt b/tests/downstream-ci-requirements.txt new file mode 100644 index 0000000..663bd1f --- /dev/null +++ b/tests/downstream-ci-requirements.txt @@ -0,0 +1 @@ +requests \ No newline at end of file From 7c95677a67048d50d5e72155eb4615b3c81be944 Mon Sep 17 00:00:00 2001 From: Sandor Kertesz Date: Wed, 11 Oct 2023 09:41:24 +0100 Subject: [PATCH 15/16] Fix check-manifest --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index b372f46..32f42de 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -9,4 +9,5 @@ recursive-include tests *.py recursive-include tests *.yml recursive-include tests *.csv recursive-include docs * +recursive-include tests *.txt From 6d1c2dae45693d551eee6840ce29442f10550e61 Mon Sep 17 00:00:00 2001 From: Iain Russell Date: Thu, 19 Oct 2023 08:22:15 +0100 Subject: [PATCH 16/16] Add labelled PR approval for ci --- .github/workflows/label-public-pr-yml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/label-public-pr-yml diff --git a/.github/workflows/label-public-pr-yml b/.github/workflows/label-public-pr-yml new file mode 100644 index 0000000..bda6c82 --- /dev/null +++ b/.github/workflows/label-public-pr-yml @@ -0,0 +1,10 @@ +# Manage labels of pull requests that originate from forks +name: label-public-pr + +on: + pull_request_target: + types: [opened, synchronize] + +jobs: + label: + uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2