From 862c5606ca78e64f5ed50d6db4572c8eb31b940a Mon Sep 17 00:00:00 2001 From: "Adam J. Jackson" Date: Wed, 11 Dec 2024 11:36:28 +0000 Subject: [PATCH] Fix and test sdist builds; reduce sdist size (#337) ## Fix bug in build_utils/version.py build_utils/version.py was raising errors in scenarios where git is unavailable. This happens more on Windows because 3 different names for git are attempted. ## Add windows sdist testing to pypi wheels workflow Wheels are already tested by cibuildwheel but a separate action is now used to test the source build. This test runs on Windows as this was the most vulnerable to the fixed bug ## Update .gitattributes: exclude tests, docs and workflows from archive meson-python uses git-archive to create the sdist. Some of euphonic's test files are on the large side and we don't need to include them with every copy of the source. This does have the downside that git-archive can no longer be used to produce a copy of the source with tests, but we weren't using that anyway. --- .gitattributes | 6 +- .../workflows/build_upload_pypi_wheels.yml | 100 +++++++++++++++--- CHANGELOG.rst | 13 +++ build_utils/version.py | 10 +- tests_and_analysis/tox_requirements.txt | 3 +- 5 files changed, 109 insertions(+), 23 deletions(-) diff --git a/.gitattributes b/.gitattributes index 6cec1af8d..77776411e 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,4 @@ -# Added by versioneer -euphonic/_version.py export-subst +.github/** export-ignore +doc/** export-ignore +tests_and_analysis/** export-ignore +tox.ini export-ignore diff --git a/.github/workflows/build_upload_pypi_wheels.yml b/.github/workflows/build_upload_pypi_wheels.yml index 83a5a5339..e5893a472 100644 --- a/.github/workflows/build_upload_pypi_wheels.yml +++ b/.github/workflows/build_upload_pypi_wheels.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - build: + build-wheels: strategy: matrix: os: [windows-latest, macos-13, macos-latest, ubuntu-latest] @@ -33,13 +33,14 @@ jobs: fail-fast: false runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 # Ensure tags are fetched for versioning + - name: Checkout project (for test files) + uses: actions/checkout@v4 + - name: Setup Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Install llvm on Macos if: startsWith(matrix.os, 'macos') shell: bash -l {0} @@ -75,7 +76,7 @@ jobs: run: | python -m pip install --upgrade pip build wheel twine - - name: Build wheels + - name: Build wheels from git checkout uses: pypa/cibuildwheel@v2.21.3 env: CIBW_BUILD_FRONTEND: build @@ -86,34 +87,99 @@ jobs: CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" CIBW_TEST_EXTRAS: "test,brille,phonopy_reader,matplotlib" - CIBW_TEST_COMMAND: python {package}/tests_and_analysis/test/run_tests.py + CIBW_TEST_COMMAND: "python {package}/tests_and_analysis/test/run_tests.py" with: output-dir: wheelhouse + - name: Upload wheels as build artifacts + uses: actions/upload-artifact@v4 + with: + name: wheel-${{ matrix.wheelname }}-${{ matrix.python-version }}-${{ matrix.cibw_archs }} + path: wheelhouse/*-${{ matrix.wheelname }}*_${{ matrix.cibw_archs }}.whl + if-no-files-found: error + + build-sdist: + name: Build sdist + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Ensure tags are fetched for versioning + - name: Create source distribution - if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' shell: bash -l {0} run: | - python -m build --sdist . + pipx run build --sdist . - name: Upload source dist as build artifact - if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' && github.event_name == 'release' uses: actions/upload-artifact@v4 with: name: python-source-distribution - path: dist + path: dist/ + if-no-files-found: error - - name: Upload wheels as build artifacts - uses: actions/upload-artifact@v4 + test-sdist: + needs: build-sdist + name: Test build from sdist on Windows + runs-on: windows-latest + steps: + - name: Setup Python + uses: actions/setup-python@v5 with: - name: wheel-${{ matrix.wheelname }}-${{ matrix.python-version }}-${{ matrix.cibw_archs }} - path: wheelhouse/*-${{ matrix.wheelname }}*_${{ matrix.cibw_archs }}.whl - if-no-files-found: error + python-version: 3.11 + + - name: Find MSVC and set environment variables + shell: bash -l {0} + env: + MSVC_PREFIX: 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\MSVC' + run: | + echo "Available MSVC installations:" + ls "$MSVC_PREFIX" + + MSVC_BIN=$(ls "$MSVC_PREFIX" | tail -n 1)\\bin\\HostX64\\x64 + CC="$MSVC_PREFIX\\$MSVC_BIN\\cl.exe" + echo "CC: $CC" + echo "CC=$CC" >> $GITHUB_ENV + + CC_LD="$MSVC_PREFIX\\$MSVC_BIN\\link.exe" + echo "CC_LD: $CC_LD" + echo "CC_LD=$CC_LD" >> $GITHUB_ENV + + - name: Download sdist + uses: actions/download-artifact@v4 + with: + path: dist/ + merge-multiple: true + + - name: List downloaded sdist + run: | + ls -R dist/ + + - name: Update pip + shell: bash -l {0} + run: | + python -m pip install --upgrade pip + + - name: Install from sdist + shell: bash -l {0} + run: python -m pip install $(find dist -name 'euphonic-*.tar.gz')[matplotlib,phonopy_reader,brille,test] + + - name: Checkout repository (for tests and test data) + uses: actions/checkout@v4 + + - name: Delete source (to ensure we are testing INSTALLED version) + shell: bash -l {0} + run: rm -rf euphonic + + - name: run tests + shell: bash -l {0} + run: python tests_and_analysis/test/run_tests.py --report publish: if: github.event_name == 'release' - needs: build + needs: [build-wheels,test-sdist] name: Upload release to PyPI runs-on: ubuntu-latest environment: @@ -134,7 +200,7 @@ jobs: merge-multiple: true - name: List Files - run: ls -R + run: ls -R dist/ - name: Upload wheels to PyPI uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 7983433ed..ab434d6f8 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,19 @@ `Unreleased `_ ------------------------------------------------------------------------------- +- Bug fixes + + - Fixed a bug in the version numbering mechanism affecting builds from sdist + on Windows and environments where Git is unavailable + + - Reduce size of sdist, removing docs and tests from + archive. (Restoring similar size to pre-v1.4.0 releases.) + +- Maintenance + + - Source builds automatically tested on Windows as part of + build/deployment process + `v1.4.0 `_ ----------------------------------------------------------------------------- diff --git a/build_utils/version.py b/build_utils/version.py index 6e2d0df95..876786f3d 100644 --- a/build_utils/version.py +++ b/build_utils/version.py @@ -39,12 +39,17 @@ for gitcmd in gits: try: + print(f"Trying {gitcmd} ...", file=sys.stderr) proc = subprocess.run([gitcmd, "describe", "--tags", "--dirty"], capture_output=True, check=True, text=True) - except subprocess.CalledProcessError as err: + except FileNotFoundError as err: + print(f"Tried {gitcmd}, File Not Found", file=sys.stderr) + continue + + except (subprocess.CalledProcessError) as err: print(f"Tried {gitcmd}, returned: {err}", file=sys.stderr) print(f"Stdout: '{err.stdout.strip()}'", file=sys.stderr) - print(f"Stdout: '{err.stderr.strip()}'", file=sys.stderr) + print(f"Stderr: '{err.stderr.strip()}'", file=sys.stderr) continue version, *dirty = proc.stdout.strip().split("-") @@ -53,6 +58,7 @@ break else: # Can't use git + print("All git implementations failed, reading version file", file=sys.stderr) version = version_file.read_text().split("=")[1].strip('"\n ') match COMMAND: diff --git a/tests_and_analysis/tox_requirements.txt b/tests_and_analysis/tox_requirements.txt index 35356c5e4..64efd55c5 100644 --- a/tests_and_analysis/tox_requirements.txt +++ b/tests_and_analysis/tox_requirements.txt @@ -1,10 +1,9 @@ mock -pytest~=7.0 +pytest==7.* coverage pytest-mock pytest-lazy-fixture pytest-xvfb python-slugify -toolz wheel==0.43 setuptools==71.1.0