ECC-1972: Python bindings: Support Windows with binary wheel (#108)

* ECC-1972: support for Windows binary wheel

.github/workflows/build-wheel-windows.yml

@@ -0,0 +1,216 @@
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+
+name: Build Windows
+
+on:
+  # Trigger the workflow manually
+  workflow_dispatch: ~
+
+  # Allow to be called from another workflow
+  workflow_call: ~
+
+  push:
+    tags-ignore:
+    - '**'
+    paths:
+    - 'scripts/common.sh'
+    - 'scripts/wheel-windows.sh'
+    - 'scripts/build-windows.sh'
+    - 'scripts/copy-dlls.py'
+    - 'scripts/copy-licences.py'
+    - '.github/workflows/build-wheel-windows.yml'
+
+
+jobs:
+
+  build:
+
+    # if: false  # for temporarily disabling for debugging
+
+    runs-on: windows-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        architecture: ["x64"]
+
+    defaults:
+      run:
+        shell: bash
+
+
+    name: Build on ${{ matrix.architecture }}
+    env:
+      WINARCH: ${{ matrix.architecture }}
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: seanmiddleditch/gha-setup-vsdevenv@master
+        with:
+          arch: ${{ matrix.architecture }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+          architecture: ${{ matrix.architecture }}
+
+      - run: ./scripts/build-windows.sh
+        env:
+          WINARCH: ${{ matrix.architecture }}
+
+
+    ################################################################
+      - name: Set up Python 3.8
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.8
+          architecture: ${{ matrix.architecture }}
+
+      - run: ./scripts/wheel-windows.sh 3.8
+      - uses: actions/upload-artifact@v4
+        name: Upload wheel 3.8
+        with:
+          name: wheel-windows-3.8-${{ matrix.architecture }}
+          path: wheelhouse/*.whl
+
+    ################################################################
+
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+          architecture: ${{ matrix.architecture }}
+
+      - run: ./scripts/wheel-windows.sh 3.9
+      - uses: actions/upload-artifact@v4
+        name: Upload wheel 3.9
+        with:
+          name: wheel-windows-3.9-${{ matrix.architecture }}
+          path: wheelhouse/*.whl
+
+    ################################################################
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          architecture: ${{ matrix.architecture }}
+
+      - run: ./scripts/wheel-windows.sh "3.10"
+      - uses: actions/upload-artifact@v4
+        name: Upload wheel 3.10
+        with:
+          name: wheel-windows-3.10-${{ matrix.architecture }}
+          path: wheelhouse/*.whl
+
+    ################################################################
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.11"
+          architecture: ${{ matrix.architecture }}
+
+      - run: ./scripts/wheel-windows.sh "3.11"
+      - uses: actions/upload-artifact@v4
+        name: Upload wheel 3.11
+        with:
+          name: wheel-windows-3.11-${{ matrix.architecture }}
+          path: wheelhouse/*.whl
+
+    ################################################################
+
+      - name: Set up Python 3.12
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.12"
+          architecture: ${{ matrix.architecture }}
+
+      - run: ./scripts/wheel-windows.sh "3.12"
+      - uses: actions/upload-artifact@v4
+        name: Upload wheel 3.12
+        with:
+          name: wheel-windows-3.12-${{ matrix.architecture }}
+          path: wheelhouse/*.whl
+
+    ################################################################
+
+
+  test:
+    needs: build
+    runs-on: windows-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        architecture: ["x64"]
+
+    defaults:
+      run:
+        shell: bash
+
+    name: Test with Python ${{ matrix.python-version }} ${{ matrix.architecture }}
+
+    steps:
+    - uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+        architecture: ${{ matrix.architecture }}
+
+    - uses: actions/download-artifact@v4
+      with:
+        name: wheel-windows-${{ matrix.python-version }}-${{ matrix.architecture }}
+
+    - run: pip install *.whl
+
+    - run: pip install -r tests/requirements.txt
+
+    - run: pip freeze
+
+    - run: ECCODES_PYTHON_TRACE_LIB_SEARCH=1 pytest --verbose -s
+      working-directory: tests
+      timeout-minutes: 2
+
+
+  deploy:
+    if: ${{ github.ref_type == 'tag' || github.event_name == 'release' }}
+
+    needs: [test, build]
+
+    name: Deploy wheel ${{ matrix.python-version }} ${{ matrix.architecture }}
+
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: true
+      matrix:
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        architecture: ["x64"]
+
+    steps:
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    - run: pip install twine
+
+    - uses: actions/download-artifact@v4
+      with:
+        name: wheel-windows-${{ matrix.python-version }}-${{ matrix.architecture }}
+
+    - run: twine upload *.whl
+      env:
+        TWINE_USERNAME: __token__
+        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

.github/workflows/cd.yml

@@ -24,8 +24,11 @@ jobs:
   wheel-macos:
     uses: ./.github/workflows/build-wheel-macos.yml
     secrets: inherit
+  wheel-windows:
+    uses: ./.github/workflows/build-wheel-windows.yml
+    secrets: inherit
   pypi:
-    needs: [wheel-linux, wheel-macos]
+    needs: [wheel-linux, wheel-macos, wheel-windows]
     uses: ecmwf-actions/reusable-workflows/.github/workflows/cd-pypi.yml@v2
     secrets: inherit
 

scripts/build-windows.sh

@@ -0,0 +1,109 @@
+
+#!/usr/bin/env bash
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation
+# nor does it submit to any jurisdiction.
+
+set -eaux
+
+source scripts/common.sh
+
+here=$(pwd)
+cd $VCPKG_INSTALLATION_ROOT
+url=$(git remote -v | head -1 | awk '{print $2;}')
+sha1=$(git rev-parse HEAD)
+cd $here
+
+echo git $url $sha1 > versions
+
+# the results of the following suggested that pkg-config.exe is not installed on the latest
+# Windows runner
+#find /c/ -name pkg-config.exe
+#exit 1
+
+# if [[ $WINARCH == "x64" ]]; then
+#     PKG_CONFIG_EXECUTABLE=/c/rtools43/mingw64/bin/pkg-config.exe
+# else
+#     PKG_CONFIG_EXECUTABLE=/c/rtools43/mingw32/bin/pkg-config.exe
+# fi
+
+vcpkg install pkgconf
+
+for p in libpng
+do
+    vcpkg install $p:$WINARCH-windows
+    n=$(echo $p | sed 's/\[.*//')
+    v=$(vcpkg list $n | awk '{print $2;}')
+    echo "vcpkg $n $v" >> versions
+done
+
+echo =================================================================
+find $VCPKG_INSTALLATION_ROOT -type f -name png.h -print
+echo =================================================================
+
+
+pip install ninja wheel dll-diagnostics
+
+echo "pip $(pip freeze | grep dll-diagnostics | sed 's/==/ /')" >> versions
+
+# Build libaec
+git clone $GIT_AEC src/aec
+cd src/aec
+git checkout $AEC_VERSION
+cd $TOPDIR
+mkdir -p build-binaries/aec
+cd build-binaries/aec
+
+cmake  \
+    $TOPDIR/src/aec -G"NMake Makefiles" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=$TOPDIR/install \
+    -DCMAKE_TOOLCHAIN_FILE=/c/vcpkg/scripts/buildsystems/vcpkg.cmake \
+    -DCMAKE_C_COMPILER=cl.exe
+
+cd $TOPDIR
+cmake --build build-binaries/aec --target install
+
+
+
+# Build eccodes
+
+cd $TOPDIR/build-binaries/eccodes
+
+$TOPDIR/src/ecbuild/bin/ecbuild \
+    $TOPDIR/src/eccodes \
+    -G"NMake Makefiles" \
+    -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+    -DENABLE_PYTHON=0 \
+    -DENABLE_FORTRAN=0 \
+    -DENABLE_BUILD_TOOLS=0 \
+    -DENABLE_MEMFS=1 \
+    -DENABLE_INSTALL_ECCODES_DEFINITIONS=0 \
+    -DENABLE_INSTALL_ECCODES_SAMPLES=0 \
+    -DCMAKE_INSTALL_PREFIX=$TOPDIR/install \
+    -DCMAKE_TOOLCHAIN_FILE=/c/vcpkg/scripts/buildsystems/vcpkg.cmake \
+    -DCMAKE_C_COMPILER=cl.exe $ECCODES_COMMON_CMAKE_OPTIONS 
+
+    # -DPKG_CONFIG_EXECUTABLE=$PKG_CONFIG_EXECUTABLE
+
+cd $TOPDIR
+cmake --build build-binaries/eccodes --target install
+
+
+# Create wheel
+
+rm -fr dist wheelhouse eccodes/share
+python scripts/copy-dlls.py install/bin/eccodes.dll eccodes/
+
+pip install -r scripts/requirements.txt
+find eccodes -name '*.dll' > libs
+cat libs
+python ./scripts/copy-licences.py libs
+
+mkdir -p install/include
+
+./scripts/versions.sh > eccodes/versions.txt

scripts/copy-dlls.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python
+#
+# (C) Copyright 2024- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+#
+
+import os
+import shutil
+import sys
+
+from dlldiag.common import ModuleHeader
+
+VCPKG1 = "C:/vcpkg/installed/{}-windows/bin/{}"
+VCPKG2 = "C:/vcpkg/installed/{}-windows/debug/bin/{}"
+
+
+def scan_module(module, depth, seen):
+    name = os.path.basename(module)
+
+    if name in seen:
+        return
+
+    if not os.path.exists(module):
+        return
+
+    print(" " * depth, module)
+    seen[name] = module
+
+    header = ModuleHeader(module)
+    cwd = os.path.dirname(module)
+    architecture = header.getArchitecture()
+    for dll in header.listAllImports():
+        # print("DEBUG", dll)
+        scan_module((cwd + "/" + dll), depth + 3, seen)
+        scan_module(VCPKG1.format(architecture, dll), depth + 3, seen)
+        scan_module(VCPKG2.format(architecture, dll), depth + 3, seen)
+
+
+seen = {}
+scan_module(sys.argv[1], 0, seen)
+
+for k, v in seen.items():
+    target = sys.argv[2] + "/" + k
+    print("Copy", v, "to", target)
+    shutil.copyfile(v, target)