diff --git a/.github/workflows/building.yml b/.github/workflows/building.yml index 2213c79e7..f5e6c8e99 100644 --- a/.github/workflows/building.yml +++ b/.github/workflows/building.yml @@ -1,10 +1,27 @@ -name: Building Wheels +name: Build Wheels -on: [workflow_dispatch] +on: [workflow_call, workflow_dispatch] jobs: - - wheel: + build_sdist: + name: Build source distribution and no binary wheel + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Strip unsupported tags in README + run: | + sed -i '//,//d' README.md + - name: Build sdist + run: BUILD_NO_CUDA=1 pipx run build --sdist + - name: Build wheel with no binaries + run: BUILD_NO_CUDA=1 python setup.py bdist_wheel --dist-dir=dist + - uses: actions/upload-artifact@v3 + with: + name: pypi_packages + path: dist/*.tar.gz + + + build_wheels: runs-on: ${{ matrix.os }} environment: production @@ -12,15 +29,18 @@ jobs: fail-fast: false matrix: os: [ubuntu-20.04, windows-2019] - python-version: ['3.8', '3.9', '3.10', '3.11', '3.12'] + python-version: ['3.10'] torch-version: ['2.0.0', '2.1.0', '2.2.0', '2.3.0', '2.4.0'] cuda-version: ['cu118', 'cu121', 'cu124'] + # python-version: ['3.8'] + # torch-version: ['2.0.0'] + # cuda-version: ['cu118'] + exclude: - python-version: 3.12 torch-version: 2.0.0 - python-version: 3.12 torch-version: 2.1.0 - - torch-version: 2.0.0 cuda-version: 'cu113' - torch-version: 2.0.0 @@ -58,21 +78,19 @@ jobs: - torch-version: 2.4.0 cuda-version: 'cu116' - torch-version: 2.4.0 - cuda-version: 'cu117' - + cuda-version: 'cu117' - os: windows-2019 torch-version: 2.0.0 cuda-version: 'cu121' - steps: - name: Checkout repository uses: actions/checkout@v4 with: submodules: recursive - + - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -146,3 +164,10 @@ jobs: python -c "import gsplat; print('gsplat:', gsplat.__version__)" cd .. shell: bash # `ls -lah` does not exist in windows powershell + + # v4 does not allow writing into the same folder, so stick with v3 + # https://github.com/actions/upload-artifact/issues/478 + - uses: actions/upload-artifact@v3 + with: + name: compiled_wheels_python${{ matrix.python-version }} + path: dist/*.whl diff --git a/.github/workflows/core_tests.yml b/.github/workflows/core_tests.yml index 7226317c9..a4db904fc 100644 --- a/.github/workflows/core_tests.yml +++ b/.github/workflows/core_tests.yml @@ -19,7 +19,7 @@ jobs: submodules: 'recursive' - name: Set up Python 3.8.12 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.8.12" - name: Install dependencies diff --git a/.github/workflows/generate_simple_index_pages.py b/.github/workflows/generate_simple_index_pages.py new file mode 100644 index 000000000..bb0537175 --- /dev/null +++ b/.github/workflows/generate_simple_index_pages.py @@ -0,0 +1,152 @@ +import requests +import os +import argparse +from jinja2 import Template +import re + +# Automatically get the repository name in the format "owner/repo" from the GitHub workflow environment +GITHUB_REPO = os.getenv("GITHUB_REPOSITORY") + + +def list_python_wheels(): + # GitHub API URL for releases + releases_url = f"https://api.github.com/repos/{GITHUB_REPO}/releases" + + response = requests.get(releases_url) + + if response.status_code != 200: + raise Exception( + f"Failed to fetch releases: {response.status_code} {response.text}" + ) + + releases = response.json() + + wheel_files = [] + + # Iterate through releases and assets + for release in releases: + assets = release.get("assets", []) + for asset in assets: + filename = asset["name"] + if filename.endswith(".whl"): + pattern = r"^(?P[\w\d_.]+)-(?P[\d.]+)(?P\+[\w\d.]+)?-(?P[\w]+)-(?P[\w]+)-(?P[\w]+)\.whl" + + match = re.match(pattern, filename) + + if match: + local_version = match.group("local") + if local_version: + local_version = local_version.lstrip( + "+" + ) # Return the local version without the '+' sign + else: + local_version = None + else: + raise ValueError(f"Invalid wheel filename: {filename}") + wheel_files.append( + { + "release_name": release["name"], + "wheel_name": asset["name"], + "download_url": asset["browser_download_url"], + "package_name": match.group("name"), + "local_version": local_version, + } + ) + + return wheel_files + + +def generate_simple_index_htmls(wheels, outdir): + # Jinja2 template as a string + template_versions_str = """ + + + Python wheels links for {{ repo_name }} + +

Python wheels for {{ repo_name }}

+ + {% for wheel in wheels %} + {{ wheel.wheel_name }}
+ {% endfor %} + + + + """ + + template_packages_str = """ + + + {% for package_name in package_names %} + {{package_name}}
+ {% endfor %} + + + """ + + # Create a Jinja2 Template object from the string + template_versions = Template(template_versions_str) + template_packages = Template(template_packages_str) + + # group the wheels by package name + packages = {} + for wheel in wheels: + package_name = wheel["package_name"] + if package_name not in packages: + packages[package_name] = [] + packages[package_name].append(wheel) + + # Render the HTML the list the package names + html_content = template_packages.render( + package_names=[str(k) for k in packages.keys()] + ) + os.makedirs(outdir, exist_ok=True) + with open(os.path.join(outdir, "index.html"), "w") as file: + file.write(html_content) + + # for each package, render the HTML to list the wheels + for package_name, wheels in packages.items(): + html_page = template_versions.render(repo_name=GITHUB_REPO, wheels=wheels) + os.makedirs(os.path.join(outdir, package_name), exist_ok=True) + with open(os.path.join(outdir, package_name, "index.html"), "w") as file: + file.write(html_page) + + +def generate_all_pages(): + wheels = list_python_wheels() + if wheels: + print("Python Wheels found in releases:") + for wheel in wheels: + print( + f"Release: {wheel['release_name']}, Wheel: {wheel['wheel_name']}, URL: {wheel['download_url']}" + ) + else: + print("No Python wheels found in the releases.") + + # Generate Simple Index HTML pages the wheel with all local versions + generate_simple_index_htmls(wheels, outdir=args.outdir) + + # group wheels per local version + wheels_per_local_version = {} + for wheel in wheels: + local_version = wheel["local_version"] + if local_version is not None and local_version not in wheels_per_local_version: + wheels_per_local_version[local_version] = [] + wheels_per_local_version[local_version].append(wheel) + + # create a subdirectory for each local version + for local_version, wheels in wheels_per_local_version.items(): + os.makedirs(os.path.join(args.outdir, local_version), exist_ok=True) + generate_simple_index_htmls( + wheels, outdir=os.path.join(args.outdir, local_version) + ) + + +if __name__ == "__main__": + argparser = argparse.ArgumentParser( + description="Generate Python Wheels Index Pages" + ) + argparser.add_argument( + "--outdir", help="Output directory for the index pages", default="." + ) + args = argparser.parse_args() + generate_all_pages() diff --git a/.github/workflows/generate_simple_index_pages.yml b/.github/workflows/generate_simple_index_pages.yml new file mode 100644 index 000000000..17b3397dd --- /dev/null +++ b/.github/workflows/generate_simple_index_pages.yml @@ -0,0 +1,42 @@ +# This workflows will upload a Python Package using twine when a release is created +# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries + +name: Update wheels index pages + +# trigger manually on github interface +on: + workflow_dispatch: + push: + branches: [generate_wheels_simple_index_pages] + +jobs: + deploy: + runs-on: ubuntu-latest + environment: production + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install dependencies + run: | + python -m pip install requests jinja2 + # call the script to generate the simple index pages + - name: Generate Simple Index Pages + run: python .github/workflows/generate_simple_index_pages.py --outdir ./whl + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./whl # Directory where the simple index pages are located + destination_dir: whl # The 'wh' folder in the GitHub Pages root \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index a163fbed1..941866cce 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,36 +1,93 @@ # This workflows will upload a Python Package using twine when a release is created # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries -name: Upload Python Package +name: Build and Release Wheels on: release: types: [created] - branches: [main] jobs: - deploy: + + # Build the wheels using reusable_building.yml + build_wheels: + name: Call reusable building workflow + uses: ./.github/workflows/building.yml + + create_release_and_upload_packages: + name: Uplodad to Github Release + needs: [build_wheels] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.10'] + steps: + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Download packages + id: download_artifacts + uses: actions/download-artifact@v3 + with: + name: compiled_wheels_python${{ matrix.python-version }} + path: dist + + - name: Upload packages to GitHub Release + id: upload_assets + run: | + for file in $(ls ./dist/*.*); do + echo "Uploading $file..." + filename=$(basename "$file") + encoded_filename=$(echo "$filename" | sed 's/+/%2B/g') + curl -X POST \ + -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Content-Type: application/zip" \ + --data-binary @"$file" \ + "${{ github.event.release.upload_url }}=$encoded_filename" + done + + generate_simple_index_pages: + name: Generate Simple Index Pages + needs: [create_release_and_upload_packages] runs-on: ubuntu-latest - environment: production + steps: + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Generate Simple Index Pages + run: python .github/workflows/generate_simple_index_pages.py --outdir ./whl + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Deploy to GitHub Pages + uses: peaceiris/actions-gh-pages@v3 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./whl # Directory where the simple index pages are located + destination_dir: whl # The 'wh' folder in the GitHub Pages root + upload_pypi: + name: Upload to PyPi + needs: [build_wheels] + runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - with: - submodules: 'recursive' - - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: '3.7' - - name: Install dependencies - run: | - python -m pip install build twine - - name: Strip unsupported tags in README - run: | - sed -i '//,//d' README.md - - name: Build and publish - env: - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - run: | - BUILD_NO_CUDA=1 python -m build - twine upload --username __token__ --password $PYPI_TOKEN dist/* \ No newline at end of file + + - uses: actions/download-artifact@v3 + with: + name: pypi_packages + path: dist + + # - name: Publish package to Test PyPI + # uses: pypa/gh-action-pypi-publish@release/v1 + # with: + # password: ${{ secrets.TEST_PYPI_API_TOKEN }} + # repository-url: https://test.pypi.org/legacy/ + + - name: Publish package to PyPI + env: + PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} + run: | + twine upload --username __token__ --password $PYPI_TOKEN dist/* \ No newline at end of file diff --git a/README.md b/README.md index 691c9cd89..fb763db1c 100644 --- a/README.md +++ b/README.md @@ -24,12 +24,24 @@ The easiest way is to install from PyPI. In this way it will build the CUDA code pip install gsplat ``` -Or install from source. In this way it will build the CUDA code during installation. +Alternatively, you can install gsplat from python wheels containing pre-compiled binaries for a specific pytorch and cuda version. These wheels are stored in the github releases and can be found using simple index pages under https://docs.gsplat.studio/whl. +You obtain the wheel from this simple index page for a specific pytorch an and cuda version by appending these the version number after a + sign (part referred a *local version*). For example, to install gsplat for pytorch 2.0 and cuda 11.8 you can use +``` +pip install gsplat==1.2.0+pt20cu118 --index-url https://docs.gsplat.studio/whl +``` +Alternatively, you can specify the pytorch and cuda version in the index url using for example +``` +pip install gsplat --index-url https://docs.gsplat.studio/whl/pt20cu118 +``` +This has the advantage that you do not have to pin a specific version of the package and as a result get automatically the latest package version. + +Alternatively you can install gsplat from source. In this way it will build the CUDA code during installation. ```bash pip install git+https://github.com/nerfstudio-project/gsplat.git ``` + To install gsplat on Windows, please check [this instruction](docs/INSTALL_WIN.md). ## Evaluation diff --git a/docs/INSTALL_WIN.md b/docs/INSTALL_WIN.md index a79366dd0..4ad098a80 100644 --- a/docs/INSTALL_WIN.md +++ b/docs/INSTALL_WIN.md @@ -1,11 +1,30 @@ # Installing `gsplat` on Windows -Follow these steps to install `gsplat` on Windows. +## Install using a pre-compiled wheels -## Prerequisites +You can install gsplat from python wheels containing pre-compiled binaries for a specific pytorch and cuda version. These wheels are stored in the github releases and can be found using simple index pages under https://docs.gsplat.studio/whl. +You obtain the wheel from this simple index page for a specific pytorch an and cuda version by appending these the version number after a + sign (part referred a *local version*). For example, to install gsplat for pytorch 2.0 and cuda 11.8 you can use +``` +pip install gsplat==1.2.0+pt20cu118 --index-url https://docs.gsplat.studio/whl +``` +Alternatively, you can specify the pytorch and cuda version in the index url using for example +``` +pip install gsplat --index-url https://docs.gsplat.studio/whl/pt20cu118 +``` +This has the advantage that you do not have to pin a specific version of the package and as a result get automatically the latest package version. + + +## Install from source + +You can install gsplat by compiling the wheel. In this way it will build the CUDA code during installation. This can be done using either the source package from pypi.org the wheel from pypi.org or using a clone of the repository. In all case Visual Studio needs to be install and activated. + +### Visual studio setup + +Setting up and activating Visual Studio can be done through these steps: 1. Install Visual Studio Build Tools. If MSVC 143 does not work, you may also need to install MSVC 142 for Visual Studio 2019. And your CUDA environment should be set up properly. + 2. Activate your conda environment: ```bash conda activate @@ -37,21 +56,37 @@ Follow these steps to install `gsplat` on Windows. ./vcvarsall.bat x64 -vcvars_ver=14.29 ``` -## Clone the Repository +### Install using the source package published on `pypi.org` -5. Clone the `gsplat` repository: +You can install gsplat from the published source package (and not the wheel) by activating Visual Studio (see above) and then using +``` +pip install --no-binary=gsplat gsplat --no-cache-dir +``` +The CUDA code will be compiled during the installation and the Visual Studio compiler `cl.exe` does not need to be added to the path, because the installation process as an automatic way to find it. +We use `--no-cache-dir` to avoid the potential risk of getting pip using a wheel file from `pypi.org` that would have be downloaded previously and that does not have the binaries. + +### Install using the wheel published on `pypi.org` + +Setting up and activating Visual Studio can be done through these steps: +You can install the `gsplat` using the wheel published on `pypi.org` by activating Visual Studio (see above) and then using +``` +pip install gsplat +``` +The wheel that does not contain the compiled CUDA binaries. The CUDA code is not compiled during the installation when using wheels and will be compiled at the first import of `gsplat` which requires the Visual Studio executable `cl.exe` to be on the path (see pre-requisite section above). + +### Install using the clone of the Repository +This can be done through these steps: +1. Clone the `gsplat` repository: ```bash git clone --recursive https://github.com/nerfstudio-project/gsplat.git ``` -6. Change into the `gsplat` directory: +2. Change into the `gsplat` directory: ```bash cd gsplat ``` - -## Install `gsplat` - -7. Install `gsplat` using pip: +3. Activate visual Studio +4. Install `gsplat` using pip: ```bash pip install . ```