Add downstream ci (#68)

* Add downstream ci
---------

Co-authored-by: Iain Russell <[email protected]>
Co-authored-by: Iain Russell <[email protected]>
Co-authored-by: Dusan Figala <[email protected]>

.github/ci-config.yml

@@ -0,0 +1,7 @@
+dependencies: |
+  ecmwf/ecbuild
+  MathisRosenhauer/libaec@master
+  ecmwf/eccodes
+dependency_branch: develop
+parallelism_factor: 8
+self_build: false

.github/ci-hpc-config.yml

@@ -0,0 +1,14 @@
+build:
+  python: '3.10'
+  modules:
+    - ninja
+  dependencies:
+    - ecmwf/ecbuild@develop
+    - ecmwf/eccodes@develop
+  python_dependencies:
+    - ecmwf/eccodes-python@develop
+  env:
+    - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
+    - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
+  parallel: 64
+  requirements: tests/downstream-ci-requirements.txt

.github/workflows/ci.yml

@@ -0,0 +1,76 @@
+name: ci
+
+on:
+  # Trigger the workflow on push to master or develop, except tag creation
+  push:
+    branches:
+      - 'master'
+      - 'develop'
+    tags-ignore:
+      - '**'
+
+  # Trigger the workflow on pull request
+  pull_request: ~
+
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Trigger after public PR approved for CI
+  pull_request_target:
+    types: [labeled]
+
+jobs:
+  # Run CI including downstream packages on self-hosted runners
+  downstream-ci:
+    name: downstream-ci
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci.yml@main
+    with:
+      pdbufr: ecmwf/pdbufr@${{ github.event.pull_request.head.sha || github.sha }}
+      codecov_upload: true
+    secrets: inherit
+
+  # # Run CI of private downstream packages on self-hosted runners
+  # private-downstream-ci:
+  #   name: private-downstream-ci
+  #   needs: [downstream-ci]
+  #   if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+  #   runs-on: ubuntu-latest
+  #   permissions:
+  #     pull-requests: write
+  #   steps:
+  #     - name: Dispatch private downstream CI
+  #       uses: ecmwf-actions/dispatch-private-downstream-ci@v1
+  #       with:
+  #         token: ${{ secrets.GH_REPO_READ_TOKEN }}
+  #         owner: ecmwf-actions
+  #         repository: private-downstream-ci
+  #         event_type: downstream-ci
+  #         payload: '{"metkit": "ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }}"}'
+
+  # Build downstream packages on HPC
+  downstream-ci-hpc:
+    name: downstream-ci-hpc
+    if: ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+    uses: ecmwf-actions/downstream-ci/.github/workflows/downstream-ci-hpc.yml@main
+    with:
+      pdbufr: ecmwf/pdbufr@${{ github.event.pull_request.head.sha || github.sha }}
+    secrets: inherit
+
+  # # Run CI of private downstream packages on HPC
+  # private-downstream-ci-hpc:
+  #   name: private-downstream-ci-hpc
+  #   needs: [downstream-ci-hpc]
+  #   if: (success() || failure()) && ${{ !github.event.pull_request.head.repo.fork && github.event.action != 'labeled' || github.event.label.name == 'approved-for-ci' }}
+  #   runs-on: ubuntu-latest
+  #   permissions:
+  #     pull-requests: write
+  #   steps:
+  #     - name: Dispatch private downstream CI
+  #       uses: ecmwf-actions/dispatch-private-downstream-ci@v1
+  #       with:
+  #         token: ${{ secrets.GH_REPO_READ_TOKEN }}
+  #         owner: ecmwf-actions
+  #         repository: private-downstream-ci
+  #         event_type: downstream-ci-hpc
+  #         payload: '{"metkit": "ecmwf/metkit@${{ github.event.pull_request.head.sha || github.sha }}"}'

.github/workflows/label-public-pr-yml

@@ -0,0 +1,10 @@
+# Manage labels of pull requests that originate from forks
+name: label-public-pr
+
+on:
+  pull_request_target:
+    types: [opened, synchronize]
+
+jobs:
+  label:
+    uses: ecmwf-actions/reusable-workflows/.github/workflows/label-pr.yml@v2

MANIFEST.in

@@ -9,4 +9,5 @@ recursive-include tests *.py
 recursive-include tests *.yml
 recursive-include tests *.csv
 recursive-include docs *
+recursive-include tests *.txt
 

setup.py

@@ -40,7 +40,7 @@ def parse_version_from(path: str) -> str:
     url="https://github.com/ecmwf/pdbufr",
     packages=setuptools.find_packages(),
     include_package_data=True,
-    install_requires=["attrs", "eccodes", "pandas"],
+    install_requires=["attrs", "eccodes", "pandas", "hypothesis"],
     extras_require={"tests": ["flake8", "pytest", "pytest-cov", "requests"]},
     zip_safe=True,
     keywords="eccodes bufr pandas",

tests/downstream-ci-requirements.txt

@@ -0,0 +1 @@
+requests

tests/test_40_sample_data.py

@@ -734,6 +734,14 @@ def test_ens_compressed() -> None:
 
 
 def test_sat_compressed_1() -> None:
+    def timestamp(s: str) -> T.Any:
+        ts = pd.Timestamp(s)
+        try:
+            ts = ts.as_unit("ns")
+        except:
+            pass
+        return ts
+
     columns = [
         "data_datetime",
         "latitude",
@@ -745,7 +753,7 @@ def test_sat_compressed_1() -> None:
     ]
 
     expected_first_row = {
-        "data_datetime": pd.Timestamp("2015-08-21 01:59:05"),
+        "data_datetime": timestamp("2015-08-21 01:59:05"),
         "latitude": -44.83389,
         "longitude": 171.1635,
         "nonCoordinateLatitude": -44.82399,
@@ -755,7 +763,7 @@ def test_sat_compressed_1() -> None:
     }
 
     expected_second_row = {
-        "data_datetime": pd.Timestamp("2015-08-21 01:59:05"),
+        "data_datetime": timestamp("2015-08-21 01:59:05"),
         "latitude": -44.83389,
         "longitude": 171.1635,
         "nonCoordinateLatitude": -44.82399,
@@ -765,7 +773,7 @@ def test_sat_compressed_1() -> None:
     }
 
     expected_12_row = {
-        "data_datetime": pd.Timestamp("2015-08-21 01:59:05"),
+        "data_datetime": timestamp("2015-08-21 01:59:05"),
         "latitude": -44.83389,
         "longitude": 171.1635,
         "nonCoordinateLatitude": -44.82399,
@@ -775,7 +783,7 @@ def test_sat_compressed_1() -> None:
     }
 
     expected_13_row = {
-        "data_datetime": pd.Timestamp("2015-08-21 01:59:06"),
+        "data_datetime": timestamp("2015-08-21 01:59:06"),
         "latitude": -44.77121,
         "longitude": 171.1515,
         "nonCoordinateLatitude": -44.76132,