Publish precompiled wheels in Github release and add wheel index

.github/workflows/building.yml

@@ -1,10 +1,31 @@
 name: Building Wheels
 
-on: [workflow_dispatch]
 
+on: [workflow_call, workflow_dispatch]
 jobs:
-
-  wheel:
+  build_sdist:
+    name: Build source distribution and no binary wheel
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Strip unsupported tags in README
+      run: |
+        sed -i '/<!-- pypi-strip -->/,/<!-- \/pypi-strip -->/d' README.md
+    - name: Build sdist
+      run: BUILD_NO_CUDA=1 pipx run build --sdist
+    - name: Build wheel with no binaries
+      run:  BUILD_NO_CUDA=1 python setup.py bdist_wheel --dist-dir=dist
+    - uses: actions/upload-artifact@v3
+      with:
+        name: pypi_packages
+        path: dist/*.tar.gz
+    - uses: actions/upload-artifact@v3
+      with:
+        name: pypi_packages
+        path: dist/*.whl
+
+
+  build_wheels:
     runs-on: ${{ matrix.os }}
     environment: production
 
@@ -15,12 +36,12 @@ jobs:
         python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         torch-version: ['2.0.0', '2.1.0', '2.2.0', '2.3.0', '2.4.0']
         cuda-version: ['cu118', 'cu121', 'cu124']
+
         exclude:
           - python-version: 3.12
             torch-version: 2.0.0
           - python-version: 3.12
             torch-version: 2.1.0
-
           - torch-version: 2.0.0
             cuda-version: 'cu113'
           - torch-version: 2.0.0
@@ -58,19 +79,17 @@ jobs:
           - torch-version: 2.4.0
             cuda-version: 'cu116'
           - torch-version: 2.4.0
-            cuda-version: 'cu117'
-
+            cuda-version: 'cu117'          
           - os: windows-2019
             torch-version: 2.0.0
             cuda-version: 'cu121'
 
-
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
         with:
           submodules: recursive
-
+  
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v4
         with:
@@ -146,3 +165,8 @@ jobs:
           python -c "import gsplat; print('gsplat:', gsplat.__version__)"
           cd ..
         shell: bash  # `ls -lah` does not exist in windows powershell
+
+      - uses: actions/upload-artifact@v3
+        with:
+          name: compiled_wheels_python${{ matrix.python-version }}
+          path: dist/*.whl

.github/workflows/generate_simple_index_pages.py

@@ -0,0 +1,152 @@
+import requests
+import os
+import argparse
+from jinja2 import Template
+import re
+
+# Automatically get the repository name in the format "owner/repo" from the GitHub workflow environment
+GITHUB_REPO = os.getenv("GITHUB_REPOSITORY")
+
+
+def list_python_wheels():
+    # GitHub API URL for releases
+    releases_url = f"https://api.github.com/repos/{GITHUB_REPO}/releases"
+
+    response = requests.get(releases_url)
+
+    if response.status_code != 200:
+        raise Exception(
+            f"Failed to fetch releases: {response.status_code} {response.text}"
+        )
+
+    releases = response.json()
+
+    wheel_files = []
+
+    # Iterate through releases and assets
+    for release in releases:
+        assets = release.get("assets", [])
+        for asset in assets:
+            filename = asset["name"]
+            if filename.endswith(".whl"):
+                pattern = r"^(?P<name>[\w\d_.]+)-(?P<version>[\d.]+)(?P<local>\+[\w\d.]+)?-(?P<python_tag>[\w]+)-(?P<abi_tag>[\w]+)-(?P<platform_tag>[\w]+)\.whl"
+
+                match = re.match(pattern, filename)
+
+                if match:
+                    local_version = match.group("local")
+                    if local_version:
+                        local_version = local_version.lstrip(
+                            "+"
+                        )  # Return the local version without the '+' sign
+                    else:
+                        local_version = None
+                else:
+                    raise ValueError(f"Invalid wheel filename: {filename}")
+                wheel_files.append(
+                    {
+                        "release_name": release["name"],
+                        "wheel_name": asset["name"],
+                        "download_url": asset["browser_download_url"],
+                        "package_name": match.group("name"),
+                        "local_version": local_version,
+                    }
+                )
+
+    return wheel_files
+
+
+def generate_simple_index_htmls(wheels, outdir):
+    # Jinja2 template as a string
+    template_versions_str = """
+    <!DOCTYPE html>
+    <html>
+    <head><title>Python wheels links for {{ repo_name }}</title></head>
+    <body>
+    <h1>Python wheels for {{ repo_name }}</h1>
+
+    {% for wheel in wheels %}
+    <a href="{{ wheel.download_url }}">{{ wheel.wheel_name }}</a><br/>
+    {% endfor %}
+
+    </body>
+    </html>
+    """
+
+    template_packages_str = """
+    <html>
+    <body>
+    {% for package_name in package_names %}
+        <a href="{{package_name}}/">{{package_name}}</a><br/>
+    {% endfor %}
+    </body>
+    </html>
+    """
+
+    # Create a Jinja2 Template object from the string
+    template_versions = Template(template_versions_str)
+    template_packages = Template(template_packages_str)
+
+    # group the wheels by package name
+    packages = {}
+    for wheel in wheels:
+        package_name = wheel["package_name"]
+        if package_name not in packages:
+            packages[package_name] = []
+        packages[package_name].append(wheel)
+
+    # Render the HTML the list the package names
+    html_content = template_packages.render(
+        package_names=[str(k) for k in packages.keys()]
+    )
+    os.makedirs(outdir, exist_ok=True)
+    with open(os.path.join(outdir, "index.html"), "w") as file:
+        file.write(html_content)
+
+    # for each package, render the HTML to list the wheels
+    for package_name, wheels in packages.items():
+        html_page = template_versions.render(repo_name=GITHUB_REPO, wheels=wheels)
+        os.makedirs(os.path.join(outdir, package_name), exist_ok=True)
+        with open(os.path.join(outdir, package_name, "index.html"), "w") as file:
+            file.write(html_page)
+
+
+def generate_all_pages():
+    wheels = list_python_wheels()
+    if wheels:
+        print("Python Wheels found in releases:")
+        for wheel in wheels:
+            print(
+                f"Release: {wheel['release_name']}, Wheel: {wheel['wheel_name']}, URL: {wheel['download_url']}"
+            )
+    else:
+        print("No Python wheels found in the releases.")
+
+    # Generate Simple Index HTML pages the wheel with all local versions
+    generate_simple_index_htmls(wheels, outdir=args.outdir)
+
+    # group wheels per local version
+    wheels_per_local_version = {}
+    for wheel in wheels:
+        local_version = wheel["local_version"]
+        if local_version not in wheels_per_local_version:
+            wheels_per_local_version[local_version] = []
+        wheels_per_local_version[local_version].append(wheel)
+
+    # create a subdirectory for each local version
+    for local_version, wheels in wheels_per_local_version.items():
+        os.makedirs(os.path.join(args.outdir, local_version), exist_ok=True)
+        generate_simple_index_htmls(
+            wheels, outdir=os.path.join(args.outdir, local_version)
+        )
+
+
+if __name__ == "__main__":
+    argparser = argparse.ArgumentParser(
+        description="Generate Python Wheels Index Pages"
+    )
+    argparser.add_argument(
+        "--outdir", help="Output directory for the index pages", default="."
+    )
+    args = argparser.parse_args()
+    generate_all_pages()

.github/workflows/generate_simple_index_pages.yml

@@ -0,0 +1,42 @@
+# This workflows will upload a Python Package using twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+name: Update wheels index pages
+
+# trigger manually on github interface
+on:
+  workflow_dispatch:
+  push:
+    branches: [generate_wheels_simple_index_pages]
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    environment: production
+
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: 'recursive'
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.x'
+
+      - name: Install dependencies
+        run: |
+          python -m pip install requests jinja2
+      # call the script to generate the simple index pages
+      - name: Generate Simple Index Pages
+        run: python .github/workflows/generate_simple_index_pages.py --outdir ./whl
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+
+
+      - name: Deploy to GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./whl  # Directory where the simple index pages are located
+          destination_dir: whl  # The 'wh' folder in the GitHub Pages root

.github/workflows/publish.yml

@@ -1,36 +1,94 @@
 # This workflows will upload a Python Package using twine when a release is created
 # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
 
-name: Upload Python Package
+name: Build and Release Wheels
 
 on:
   release:
     types: [created]
-    branches: [main]
 
 jobs:
-  deploy:
+
+  # Build the wheels using reusable_building.yml
+  build_wheels:
+    name: Call reusable building workflow
+    uses: ./.github/workflows/building.yml  
+
+  create_release_and_upload_packages:
+    name: Uplodad to Github Release
+    needs: [build_wheels]
     runs-on: ubuntu-latest
-    environment: production
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.10', '3.11', '3.12']
+    steps:
+
+    - name: Checkout code
+      uses: actions/checkout@v3
 
+    - name: Download packages
+      id: download_artifacts
+      uses: actions/download-artifact@v3
+      with:
+        name: compiled_wheels_python${{ matrix.python-version }}
+        path: dist
+
+    - name: Upload packages to GitHub Release
+      id: upload_assets
+      run: |
+        for file in $(ls ./dist/*.*); do
+          echo "Uploading $file..."
+          filename=$(basename "$file")
+          encoded_filename=$(echo "$filename" | sed 's/+/%2B/g')
+          curl -X POST \
+            -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \
+            -H "Content-Type: application/zip" \
+            --data-binary @"$file" \
+            "${{ github.event.release.upload_url }}=$encoded_filename"
+        done
+
+  generate_simple_index_pages:
+    name: Generate Simple Index Pages
+    needs: [create_release_and_upload_packages]
+    runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: 'recursive'
-
-      - name: Set up Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: '3.7'
-      - name: Install dependencies
-        run: |
-          python -m pip install build twine
-      - name: Strip unsupported tags in README
-        run: |
-          sed -i '/<!-- pypi-strip -->/,/<!-- \/pypi-strip -->/d' README.md
-      - name: Build and publish
-        env:
-          PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+    - name: Generate Simple Index Pages
+      run: python .github/workflows/generate_simple_index_pages.py --outdir ./wheels
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  
+
+    - name: Deploy to GitHub Pages
+      uses: peaceiris/actions-gh-pages@v3
+      with:
+        github_token: ${{ secrets.GITHUB_TOKEN }}
+        publish_dir: ./wheels  # Directory where the simple index pages are located
+        destination_dir: wheels  # The 'wh' folder in the GitHub Pages root
+
+  upload_pypi:
+    name: Upload to PyPi
+    needs: [build_wheels]
+    runs-on: ubuntu-latest
+    steps:
+
+    - uses: actions/download-artifact@v2
+      with:
+        name: pypi_packages
+        path: dist
+
+    - name: Publish package to Test PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
+        repository-url: https://test.pypi.org/legacy/
+
+    - name: Publish package to PyPI
+      env:
+        PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }}
+      run: |         
+        twine upload --username __token__ --password $PYPI_TOKEN dist/*
+
+              - name: Install dependencies
         run: |
-          BUILD_NO_CUDA=1 python -m build
-          twine upload --username __token__ --password $PYPI_TOKEN dist/*
+          python -m pip install requests jinja2
+