From 454209be35845e476e2e2a9d959d2d7f0707f39f Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 22 Aug 2023 11:19:12 +0200 Subject: [PATCH 1/3] Rework the release script entirely. --- .github/workflows/python-release.yml | 105 +++++++++++++++------------ 1 file changed, 60 insertions(+), 45 deletions(-) diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index a1a1456b..85dc026c 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -1,8 +1,8 @@ name: Python Release on: push: - tags: - - v* + # tags: + # - v* env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} @@ -27,23 +27,33 @@ jobs: working-directory: ./bindings/python run: sh build-wheels.sh - create_wheels_windows_32bit: - name: Create wheels for windows 32-bit + create_wheels_windows: + name: Windows runs-on: windows-latest strategy: matrix: python: ["3.7", "3.8", "3.9", "3.10", "3.11"] + bits: ["32", "64"] steps: - name: Checkout repository uses: actions/checkout@v2 - - name: Install Rust + - name: Install Rust 32bits + if: ${{ matrix.os == '32' }} + uses: actions-rs/toolchain@v1 + with: + toolchain: stable-i686-pc-windows-msvc + override: true + + - name: Install Rust 64bits + if: ${{ matrix.os == '32' }} uses: actions-rs/toolchain@v1 with: toolchain: stable-i686-pc-windows-msvc override: true - name: Override toolchain + if: ${{ matrix.os == '32' }} shell: bash working-directory: ./bindings/python run: echo "stable-i686-pc-windows-msvc" > rust-toolchain @@ -57,7 +67,7 @@ jobs: - name: Install dependencies run: | # On old versions of python there is an old version of setuptools already installed - pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall + pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall - name: Build wheel working-directory: ./bindings/python @@ -69,45 +79,50 @@ jobs: pip install awscli aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" - - create_wheels_others_64bit: - name: Other OSes - env: - MACOSX_DEPLOYMENT_TARGET: 10.11 + create_wheels_macos_conda: + name: MacOS - Conda runs-on: ${{ matrix.os }} strategy: matrix: - os: [windows-latest, macos-latest] - python: ["3.7", "3.8", "3.9", "3.10", "3.11"] + os: [macos-latest] + # 3.11 not available on Conda yet. + python: ["3.7", "3.8", "3.9", "3.10"] + steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 + + - name: Install miniconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-update-conda: true + python-version: ${{ matrix.python }} + + - name: Conda info + shell: bash -l {0} + run: conda info - name: Install Rust uses: actions-rs/toolchain@v1 with: toolchain: stable - override: true - - name: Install Python - uses: actions/setup-python@v4 - with: - python-version: ${{ matrix.python }} - architecture: x64 - - - name: Install dependencies + - name: Setup conda env + shell: bash -l {0} run: | - # On old versions of python there is an old version of setuptools already installed - pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall + conda install setuptools-rust + conda install -c defaults anaconda-client conda-build - - name: Build wheel + - name: Extract version + shell: bash -l {0} working-directory: ./bindings/python - run: python setup.py bdist_wheel + run: echo "SAFETENSORS_VERSION=`python setup.py --version`" >> $GITHUB_ENV - - name: Rename wheels - shell: bash - working-directory: ./bindings/python/dist - run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done + - name: Build conda packages + shell: bash -l {0} + working-directory: ./bindings/python + run: | + MACOSX_DEPLOYMENT_TARGET=10.11 python setup.py bdist_wheel - name: Upload wheels shell: bash @@ -115,13 +130,13 @@ jobs: pip install awscli aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" - create_wheels_macos_13: - name: MacOS 13 - runs-on: ${{ matrix.os }} + create_wheels_macos: + name: MacOS + runs-on: ${{ matrix.os.os }} strategy: matrix: - os: [macos-13] python: ["3.7", "3.8", "3.9", "3.10", "3.11"] + os: [{os: "macos-11", target: "11.0"}, {os: "macos-12"}, {os: "macos-13"}, {os: "macos-13", target: "14.0"}] steps: - name: Checkout repository uses: actions/checkout@v2 @@ -136,21 +151,20 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.python }} - architecture: x64 - name: Install dependencies run: | # On old versions of python there is an old version of setuptools already installed - pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall + pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall - - name: Build wheel + - name: Override target + if: ${{ matrix.os.target }} working-directory: ./bindings/python - run: python setup.py bdist_wheel + run: echo "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.target }}" >> $GITHUB_ENV - - name: Rename wheels - shell: bash - working-directory: ./bindings/python/dist - run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done + - name: Build wheel + working-directory: ./bindings/python + run: echo $MACOX_DEPLOYMENT_TARGET && python setup.py bdist_wheel - name: Upload wheels shell: bash @@ -158,13 +172,14 @@ jobs: pip install awscli aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR" + create_wheels_macos_arm64: - name: Create wheels for MacOS M1 + name: MacOS M1 runs-on: macos-arm64 strategy: matrix: python: ["3.9.13", "3.10.6", "3.11.0"] - target: ["12.0", "13.0"] + target: ["12.0", "13.0", "14.0"] steps: - name: Checkout repository uses: actions/checkout@v3 @@ -194,7 +209,7 @@ jobs: Upload_package: name: Upload package to PyPi runs-on: ubuntu-latest - needs: [create_wheels_manylinux, create_wheels_windows_32bit, create_wheels_others_64bit, create_wheels_macos_arm64, create_wheels_macos_13] + needs: [create_wheels_manylinux, create_wheels_windows, create_wheels_macos, create_wheels_macos_arm64, create_wheels_macos_conda] steps: - uses: actions/checkout@v3 From 9ee1126de4b282156ab598280dc42d903cda82fd Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 22 Aug 2023 11:23:13 +0200 Subject: [PATCH 2/3] New release script. --- .github/workflows/python-release.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index 85dc026c..f32fad34 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -12,9 +12,18 @@ env: DIST_DIR: ${{ github.sha }} jobs: + lock_exists: + runs-on: ubuntu-latest + name: Cargo.lock + steps: + - uses: actions/checkout@v3 + - name: Cargo.lock lock exists + run: cat Cargo.lock + working-directory: ./bindings/python create_wheels_manylinux: runs-on: ubuntu-latest + needs: [lock_exists] name: Create wheels for manylinux2014 container: quay.io/pypa/manylinux2014_x86_64 steps: @@ -30,6 +39,7 @@ jobs: create_wheels_windows: name: Windows runs-on: windows-latest + needs: [lock_exists] strategy: matrix: python: ["3.7", "3.8", "3.9", "3.10", "3.11"] @@ -82,6 +92,7 @@ jobs: create_wheels_macos_conda: name: MacOS - Conda runs-on: ${{ matrix.os }} + needs: [lock_exists] strategy: matrix: os: [macos-latest] @@ -133,6 +144,7 @@ jobs: create_wheels_macos: name: MacOS runs-on: ${{ matrix.os.os }} + needs: [lock_exists] strategy: matrix: python: ["3.7", "3.8", "3.9", "3.10", "3.11"] @@ -176,6 +188,7 @@ jobs: create_wheels_macos_arm64: name: MacOS M1 runs-on: macos-arm64 + needs: [lock_exists] strategy: matrix: python: ["3.9.13", "3.10.6", "3.11.0"] From 01d3094e6c29ac6df3d11a219101816d46defcc9 Mon Sep 17 00:00:00 2001 From: Nicolas Patry Date: Tue, 22 Aug 2023 11:24:57 +0200 Subject: [PATCH 3/3] Ok. --- .github/workflows/python-release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml index f32fad34..35a38ea2 100644 --- a/.github/workflows/python-release.yml +++ b/.github/workflows/python-release.yml @@ -1,8 +1,8 @@ name: Python Release on: push: - # tags: - # - v* + tags: + - v* env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}