Fix and test sdist builds; reduce sdist size (#337)

## Fix bug in build_utils/version.py

build_utils/version.py was raising errors in scenarios where git is unavailable. This happens more on Windows because 3 different names for git are attempted.

## Add windows sdist testing to pypi wheels workflow

Wheels are already tested by cibuildwheel but a separate action is now used to test the source build.

This test runs on Windows as this was the most vulnerable to the fixed bug

## Update .gitattributes: exclude tests, docs and workflows from archive

meson-python uses git-archive to create the sdist. Some of euphonic's
test files are on the large side and we don't need to include them
with every copy of the source.

This does have the downside that git-archive can no longer be used
to produce a copy of the source with tests, but we weren't using that
anyway.

.gitattributes

@@ -1,2 +1,4 @@
-# Added by versioneer
-euphonic/_version.py export-subst
+.github/** export-ignore
+doc/** export-ignore
+tests_and_analysis/** export-ignore
+tox.ini export-ignore

.github/workflows/build_upload_pypi_wheels.yml

@@ -6,7 +6,7 @@ on:
   workflow_dispatch:
 
 jobs:
-  build:
+  build-wheels:
     strategy:
       matrix:
         os: [windows-latest, macos-13, macos-latest, ubuntu-latest]
@@ -33,13 +33,14 @@ jobs:
       fail-fast: false
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v4
-        with:
-          fetch-depth: 0 # Ensure tags are fetched for versioning
+      - name: Checkout project (for test files)
+        uses: actions/checkout@v4
+
       - name: Setup Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
+
       - name: Install llvm on Macos
         if: startsWith(matrix.os, 'macos')
         shell: bash -l {0}
@@ -75,7 +76,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip build wheel twine
 
-      - name: Build wheels
+      - name: Build wheels from git checkout
         uses: pypa/[email protected]
         env:
           CIBW_BUILD_FRONTEND: build
@@ -86,34 +87,99 @@ jobs:
           CIBW_REPAIR_WHEEL_COMMAND_MACOS: ""
 
           CIBW_TEST_EXTRAS: "test,brille,phonopy_reader,matplotlib"
-          CIBW_TEST_COMMAND: python {package}/tests_and_analysis/test/run_tests.py
+          CIBW_TEST_COMMAND: "python {package}/tests_and_analysis/test/run_tests.py"
 
         with:
           output-dir: wheelhouse
 
+      - name: Upload wheels as build artifacts
+        uses: actions/upload-artifact@v4
+        with:
+            name: wheel-${{ matrix.wheelname }}-${{ matrix.python-version }}-${{ matrix.cibw_archs }}
+            path: wheelhouse/*-${{ matrix.wheelname }}*_${{ matrix.cibw_archs }}.whl
+            if-no-files-found: error
+
+  build-sdist:
+    name: Build sdist
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Ensure tags are fetched for versioning
+
       - name: Create source distribution
-        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11'
         shell: bash -l {0}
         run: |
-          python -m build --sdist .
+          pipx run build --sdist .
 
       - name: Upload source dist as build artifact
-        if: matrix.os == 'ubuntu-latest' && matrix.python-version == '3.11' && github.event_name == 'release'
         uses: actions/upload-artifact@v4
         with:
             name: python-source-distribution
-            path: dist
+            path: dist/
+            if-no-files-found: error
 
-      - name: Upload wheels as build artifacts
-        uses: actions/upload-artifact@v4
+  test-sdist:
+    needs: build-sdist
+    name: Test build from sdist on Windows
+    runs-on: windows-latest
+    steps:
+      - name: Setup Python
+        uses: actions/setup-python@v5
         with:
-            name: wheel-${{ matrix.wheelname }}-${{ matrix.python-version }}-${{ matrix.cibw_archs }}
-            path: wheelhouse/*-${{ matrix.wheelname }}*_${{ matrix.cibw_archs }}.whl
-            if-no-files-found: error
+          python-version: 3.11
+
+      - name: Find MSVC and set environment variables
+        shell: bash -l {0}
+        env:
+          MSVC_PREFIX: 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Tools\MSVC'
+        run: |
+          echo "Available MSVC installations:"
+          ls "$MSVC_PREFIX"
+
+          MSVC_BIN=$(ls "$MSVC_PREFIX" | tail -n 1)\\bin\\HostX64\\x64
+          CC="$MSVC_PREFIX\\$MSVC_BIN\\cl.exe"
+          echo "CC: $CC"
+          echo "CC=$CC" >> $GITHUB_ENV
+
+          CC_LD="$MSVC_PREFIX\\$MSVC_BIN\\link.exe"
+          echo "CC_LD: $CC_LD"
+          echo "CC_LD=$CC_LD" >> $GITHUB_ENV
+
+      - name: Download sdist
+        uses: actions/download-artifact@v4
+        with:
+          path: dist/
+          merge-multiple: true
+
+      - name: List downloaded sdist
+        run: |
+          ls -R dist/
+
+      - name: Update pip
+        shell: bash -l {0}
+        run: |
+          python -m pip install --upgrade pip
+
+      - name: Install from sdist
+        shell: bash -l {0}
+        run: python -m pip install $(find dist -name 'euphonic-*.tar.gz')[matplotlib,phonopy_reader,brille,test]
+
+      - name: Checkout repository (for tests and test data)
+        uses: actions/checkout@v4
+
+      - name: Delete source (to ensure we are testing INSTALLED version)
+        shell: bash -l {0}
+        run: rm -rf euphonic
+
+      - name: run tests
+        shell: bash -l {0}
+        run: python tests_and_analysis/test/run_tests.py --report
 
   publish:
     if: github.event_name == 'release'
-    needs: build
+    needs: [build-wheels,test-sdist]
     name: Upload release to PyPI
     runs-on: ubuntu-latest
     environment:
@@ -134,7 +200,7 @@ jobs:
           merge-multiple: true
 
       - name: List Files
-        run: ls -R
+        run: ls -R dist/
 
       - name: Upload wheels to PyPI
         uses: pypa/gh-action-pypi-publish@release/v1

CHANGELOG.rst

@@ -1,6 +1,19 @@
 `Unreleased <https://github.com/pace-neutrons/Euphonic/compare/v1.4.0...HEAD>`_
 -------------------------------------------------------------------------------
 
+- Bug fixes
+
+  - Fixed a bug in the version numbering mechanism affecting builds from sdist
+    on Windows and environments where Git is unavailable
+
+  - Reduce size of sdist, removing docs and tests from
+    archive. (Restoring similar size to pre-v1.4.0 releases.)
+
+- Maintenance
+
+  - Source builds automatically tested on Windows as part of
+    build/deployment process
+
 `v1.4.0 <https://github.com/pace-neutrons/Euphonic/compare/v1.3.2...v1.4.0>`_
 -----------------------------------------------------------------------------
 

build_utils/version.py

@@ -39,12 +39,17 @@
 
 for gitcmd in gits:
     try:
+        print(f"Trying {gitcmd} ...", file=sys.stderr)
         proc = subprocess.run([gitcmd, "describe", "--tags", "--dirty"],
                               capture_output=True, check=True, text=True)
-    except subprocess.CalledProcessError as err:
+    except FileNotFoundError as err:
+        print(f"Tried {gitcmd}, File Not Found", file=sys.stderr)
+        continue
+
+    except (subprocess.CalledProcessError) as err:
         print(f"Tried {gitcmd}, returned: {err}", file=sys.stderr)
         print(f"Stdout: '{err.stdout.strip()}'", file=sys.stderr)
-        print(f"Stdout: '{err.stderr.strip()}'", file=sys.stderr)
+        print(f"Stderr: '{err.stderr.strip()}'", file=sys.stderr)
         continue
 
     version, *dirty = proc.stdout.strip().split("-")
@@ -53,6 +58,7 @@
     break
 
 else:  # Can't use git
+    print("All git implementations failed, reading version file", file=sys.stderr)
     version = version_file.read_text().split("=")[1].strip('"\n ')
 
 match COMMAND:

tests_and_analysis/tox_requirements.txt

@@ -1,10 +1,9 @@
 mock
-pytest~=7.0
+pytest==7.*
 coverage
 pytest-mock
 pytest-lazy-fixture
 pytest-xvfb
 python-slugify
-toolz
 wheel==0.43
 setuptools==71.1.0