From c8112f49b5f28994f22dea4f4e1161ca11c86090 Mon Sep 17 00:00:00 2001 From: Yurij Mikhalevich Date: Sun, 20 Aug 2023 23:44:31 +0400 Subject: [PATCH] ci: automate hombrew releases (#66) --- .github/workflows/release.yaml | 34 +++++++++ Makefile | 4 ++ poetry.lock | 33 ++++++++- pyproject.toml | 4 +- release-utils/homebrew/generate_formula.py | 81 ++++++++++++++++++++++ release-utils/homebrew/release.sh | 51 ++++++++++++++ snap/snapcraft.yaml | 4 +- 7 files changed, 207 insertions(+), 4 deletions(-) create mode 100644 release-utils/homebrew/generate_formula.py create mode 100755 release-utils/homebrew/release.sh diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e3285d8a..4ba018d5 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -23,6 +23,40 @@ jobs: - run: poetry build - uses: pypa/gh-action-pypi-publish@release/v1 + brew: + runs-on: ubuntu-22.04 + # the Formula references the rclip package published to PyPI + needs: pypi + steps: + - name: Parse the Version + id: version + run: | + echo ::set-output name=PRERELEASE::$( + [[ ! $GITHUB_REF =~ ^refs\/tags\/v[0-9]+\.[0-9]+\.[0-9]+$ ]] && echo "true" + ) + - uses: actions/checkout@v3 + if: ${{ steps.version.outputs.PRERELEASE != 'true' }} + - uses: actions/setup-python@v4 + if: ${{ steps.version.outputs.PRERELEASE != 'true' }} + with: + python-version: 3.8 + - name: Install dependencies + if: ${{ steps.version.outputs.PRERELEASE != 'true' }} + run: | + python -m pip install --upgrade pip + pip install --upgrade poetry + poetry install + - name: Setup git + if: ${{ steps.version.outputs.PRERELEASE != 'true' }} + run: | + git config --global user.email "zhibot.gh@gmail.com" + git config --global user.name "Zhi Bot" + - name: Release + if: ${{ steps.version.outputs.PRERELEASE != 'true' }} + env: + GITHUB_TOKEN: ${{ secrets.ZHIBOT_GITHUB_TOKEN }} + run: make release-brew + snap: runs-on: ubuntu-20.04 steps: diff --git a/Makefile b/Makefile index 0e9d54f1..aa064309 100644 --- a/Makefile +++ b/Makefile @@ -25,6 +25,10 @@ test: build-docker: DOCKER_DEFAULT_PLATFORM=linux/amd64 docker build . -t rclip +# CI runs release-brew as part of the `release` action +release-brew: + poetry run ./release-utils/homebrew/release.sh + release: @test $(VERSION) || (echo "VERSION arg is required" && exit 1) poetry version $(VERSION) diff --git a/poetry.lock b/poetry.lock index 75b70dfb..53d3f73b 100644 --- a/poetry.lock +++ b/poetry.lock @@ -184,6 +184,21 @@ files = [ [package.dependencies] wcwidth = ">=0.2.5" +[[package]] +name = "homebrew-pypi-poet" +version = "0.10.0" +description = "Writes Homebrew stanzas for pypi packages" +optional = false +python-versions = "*" +files = [ + {file = "homebrew-pypi-poet-0.10.0.tar.gz", hash = "sha256:e09e997e35a98f66445f9a39ccb33d6d93c5cd090302a59f231707eac0bf378e"}, + {file = "homebrew_pypi_poet-0.10.0-py2.py3-none-any.whl", hash = "sha256:65824f97aea0e713c4ac18aa2ef4477aca69426554eac842eeaaddf97df3fc47"}, +] + +[package.dependencies] +jinja2 = "*" +setuptools = "*" + [[package]] name = "huggingface-hub" version = "0.16.4" @@ -854,6 +869,22 @@ files = [ {file = "sentencepiece-0.1.99.tar.gz", hash = "sha256:189c48f5cb2949288f97ccdb97f0473098d9c3dcf5a3d99d4eabe719ec27297f"}, ] +[[package]] +name = "setuptools" +version = "68.1.0" +description = "Easily download, build, install, upgrade, and uninstall Python packages" +optional = false +python-versions = ">=3.8" +files = [ + {file = "setuptools-68.1.0-py3-none-any.whl", hash = "sha256:e13e1b0bc760e9b0127eda042845999b2f913e12437046e663b833aa96d89715"}, + {file = "setuptools-68.1.0.tar.gz", hash = "sha256:d59c97e7b774979a5ccb96388efc9eb65518004537e85d52e81eaee89ab6dd91"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] + [[package]] name = "sympy" version = "1.12" @@ -1099,4 +1130,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "066328f767a4d1b26e27bfeec0f9df4dc0a719126da5948d2f5c3ce33b706725" +content-hash = "52936388df81d3bf44dc2215fdbdb44ab4c2fa1bb56fa1aee4767a1f3c1b671a" diff --git a/pyproject.toml b/pyproject.toml index 25c1f6aa..61568d56 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "rclip" -version = "1.5.0a0" +version = "1.5.0a7" description = "AI-Powered Command-Line Photo Search Tool" authors = ["Yurij Mikhalevich "] license = "MIT" @@ -35,6 +35,8 @@ tqdm = "^4.65.0" [tool.poetry.group.dev.dependencies] pycodestyle = ">=2.7,<3.0" pytest = ">=7.2.1,<8.0" +homebrew-pypi-poet = "^0.10.0" +jinja2 = "^3.1.2" [tool.poetry.scripts] rclip = "rclip.main:main" diff --git a/release-utils/homebrew/generate_formula.py b/release-utils/homebrew/generate_formula.py new file mode 100644 index 00000000..c6a1bfb4 --- /dev/null +++ b/release-utils/homebrew/generate_formula.py @@ -0,0 +1,81 @@ +import hashlib +import jinja2 +import poet +import requests + +env = jinja2.Environment(trim_blocks=True) + + +TEMPLATE = env.from_string('''class Rclip < Formula + include Language::Python::Virtualenv + + desc "AI-Powered Command-Line Photo Search Tool" + homepage "https://github.com/yurijmikhalevich/rclip" + url "{{ package.url }}" + sha256 "{{ package.checksum }}" + license "MIT" + + depends_on "rust" => :build # for safetensors + depends_on "numpy" + depends_on "pillow" + depends_on "python-certifi" + depends_on "python@3.11" + depends_on "pytorch" + depends_on "sentencepiece" + depends_on "torchvision" + +{{ resources }} + + def install + virtualenv_install_with_resources + + # link dependent virtualenvs to this one + site_packages = Language::Python.site_packages("python3.11") + paths = %w[pytorch torchvision].map do |package_name| + package = Formula[package_name].opt_libexec + package/site_packages + end + (libexec/site_packages/"homebrew-deps.pth").write paths.join("\\n") + end + + test do + output = shell_output("#{bin}/rclip cat") + assert_match("score\\tfilepath", output) + end +end +''') + + +# These deps are being installed from brew +DEPS_TO_IGNORE = ['numpy', 'pillow', 'certifi', 'torch', 'torchvision'] +RESOURCE_URL_OVERRIDES = { + # open-clip-torch publishes an incomplete tarball to pypi, so we will fetch one from GitHub + 'open-clip-torch': env.from_string( + 'https://github.com/mlfoundations/open_clip/archive/refs/tags/v{{ version }}.tar.gz' + ), +} + + +def main(): + deps = poet.make_graph('rclip') + for dep in DEPS_TO_IGNORE: + deps.pop(dep, None) + for dep, url in RESOURCE_URL_OVERRIDES.items(): + new_url = url.render(version=deps[dep]['version']) + deps[dep]['url'] = new_url + deps[dep]['checksum'] = compute_checksum(new_url) + for _, dep in deps.items(): + dep["name"] = dep["name"].lower() + + rclip_metadata = deps.pop('rclip') + resources = '\n\n'.join([poet.RESOURCE_TEMPLATE.render(resource=dep) for dep in deps.values()]) + print(TEMPLATE.render(package=rclip_metadata, resources=resources)) + + +def compute_checksum(url: str): + response = requests.get(url) + return hashlib.sha256(response.content).hexdigest() + + +if __name__ == '__main__': + main() diff --git a/release-utils/homebrew/release.sh b/release-utils/homebrew/release.sh new file mode 100755 index 00000000..6a35cd70 --- /dev/null +++ b/release-utils/homebrew/release.sh @@ -0,0 +1,51 @@ +#!/bin/bash +set -e + +# the script requires gh cli and git to be installed and configured +# and push permissions to https://github.com/yurijmikhalevich/homebrew-tap + +ORIG_PWD=$(pwd) +VERSION=$(poetry version -s) + +TMP_DIR=$(mktemp -d -t release-rclip-brew-XXXXXXXXXX) +cd $TMP_DIR +echo "Working in $TMP_DIR" + +function handle_exit() { + cd "$ORIG_PWD" + rm -rf $TMP_DIR + echo "Removed $TMP_DIR" +} +trap handle_exit 0 SIGHUP SIGINT SIGQUIT SIGABRT SIGTERM + +if [[ "$GITHUB_ACTIONS" ]]; then + git clone "https://$GITHUB_TOKEN@github.com/yurijmikhalevich/homebrew-tap.git" homebrew-tap +else + git clone git@github.com:yurijmikhalevich/homebrew-tap.git homebrew-tap +fi +cd homebrew-tap + +PR_BRANCH="rclip-$VERSION" +PR_TITLE="rclip $VERSION" + +git checkout -b "$PR_BRANCH" +python "$ORIG_PWD/release-utils/homebrew/generate_formula.py" > Formula/rclip.rb +git commit -am "$PR_TITLE" +git push origin "$PR_BRANCH" +gh pr create --title "$PR_TITLE" --body "Automated commit updating **rclip** formula to $VERSION" --base main --head "$PR_BRANCH" +# it takes a few seconds for GHA to start checks on the PR +sleep 20 +gh pr checks "$PR_BRANCH" --watch --fail-fast +gh pr edit "$PR_BRANCH" --add-label pr-pull +# it takes a few seconds for GHA to start checks on the PR +sleep 20 +gh pr checks "$PR_BRANCH" --watch --fail-fast + +# assert that PR_STATE was closed as it should +PR_STATE=$(gh pr view "$PR_BRANCH" --json state -q .state) +if [ "$PR_STATE" != "CLOSED" ]; then + echo "PR \"$PR_TITLE\" is not closed" + exit 1 +fi + +echo "Released rclip $VERSION" diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index dc9925dd..6d4c4a0c 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -10,7 +10,7 @@ description: | For a detailed demonstration, watch the video: https://www.youtube.com/watch?v=tAJHXOkHidw. You can use another image as a query by passing a file path or even an URL to the image file to **rclip** and combine multiple queries. Check out the project's README on GitHub for more usage examples: https://github.com/yurijmikhalevich/rclip#readme. -version: 1.5.0a0 +version: 1.5.0a7 website: https://github.com/yurijmikhalevich/rclip contact: yurij@mikhalevi.ch passthrough: @@ -33,7 +33,7 @@ apps: parts: rclip: plugin: python - source: ./snap/local/rclip-1.5.0a0.tar.gz + source: ./snap/local/rclip-1.5.0a7.tar.gz build-packages: - python3-pip build-environment: