Reworked the release script entirely (#334)

* Rework the release script entirely. * New release script. * Ok.
huggingface · Aug 22, 2023 · 2695e7b · 2695e7b
1 parent 34a2a0a
commit 2695e7b
Showing 1 changed file with 71 additions and 43 deletions.
diff --git a/.github/workflows/python-release.yml b/.github/workflows/python-release.yml
@@ -12,9 +12,18 @@ env:
   DIST_DIR: ${{ github.sha }}
 
 jobs:
+  lock_exists:
+    runs-on: ubuntu-latest
+    name: Cargo.lock
+    steps:
+      - uses: actions/checkout@v3
+      - name: Cargo.lock lock exists 
+        run: cat Cargo.lock
+        working-directory: ./bindings/python
 
   create_wheels_manylinux:
     runs-on: ubuntu-latest
+    needs: [lock_exists]
     name: Create wheels for manylinux2014
     container: quay.io/pypa/manylinux2014_x86_64
     steps:
@@ -27,23 +36,34 @@ jobs:
         working-directory: ./bindings/python
         run: sh build-wheels.sh
 
-  create_wheels_windows_32bit:
-    name: Create wheels for windows 32-bit
+  create_wheels_windows:
+    name: Windows 
     runs-on: windows-latest
+    needs: [lock_exists]
     strategy:
       matrix:
         python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        bits: ["32", "64"]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
 
-      - name: Install Rust
+      - name: Install Rust 32bits
+        if: ${{ matrix.os == '32' }}
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable-i686-pc-windows-msvc
+          override: true
+
+      - name: Install Rust 64bits
+        if: ${{ matrix.os == '32' }}
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable-i686-pc-windows-msvc
           override: true
 
       - name: Override toolchain
+        if: ${{ matrix.os == '32' }}
         shell: bash
         working-directory: ./bindings/python
         run: echo "stable-i686-pc-windows-msvc" > rust-toolchain
@@ -57,7 +77,7 @@ jobs:
       - name: Install dependencies
         run: |
           # On old versions of python there is an old version of setuptools already installed
-          pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall
+          pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall
 
       - name: Build wheel
         working-directory: ./bindings/python
@@ -69,59 +89,66 @@ jobs:
           pip install awscli
           aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR"
 
-
-  create_wheels_others_64bit:
-    name: Other OSes
-    env:
-      MACOSX_DEPLOYMENT_TARGET: 10.11
+  create_wheels_macos_conda:
+    name: MacOS - Conda
     runs-on: ${{ matrix.os }}
+    needs: [lock_exists]
     strategy:
       matrix:
-        os: [windows-latest, macos-latest]
-        python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        os: [macos-latest]
+        # 3.11 not available on Conda yet.
+        python: ["3.7", "3.8", "3.9", "3.10"]
+
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
+
+      - name: Install miniconda
+        uses: conda-incubator/setup-miniconda@v2
+        with:
+          auto-update-conda: true
+          python-version: ${{ matrix.python }}
+
+      - name: Conda info
+        shell: bash -l {0}
+        run: conda info
 
       - name: Install Rust
         uses: actions-rs/toolchain@v1
         with:
           toolchain: stable
-          override: true
 
-      - name: Install Python
-        uses: actions/setup-python@v4
-        with:
-          python-version: ${{ matrix.python }}
-          architecture: x64
-
-      - name: Install dependencies
+      - name: Setup conda env
+        shell: bash -l {0}
         run: |
-          # On old versions of python there is an old version of setuptools already installed
-          pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall
+          conda install setuptools-rust
+          conda install -c defaults anaconda-client conda-build
 
-      - name: Build wheel
+      - name: Extract version
+        shell: bash -l {0}
         working-directory: ./bindings/python
-        run: python setup.py bdist_wheel
+        run: echo "SAFETENSORS_VERSION=`python setup.py --version`" >> $GITHUB_ENV
 
-      - name: Rename wheels
-        shell: bash
-        working-directory: ./bindings/python/dist
-        run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done
+      - name: Build conda packages
+        shell: bash -l {0}
+        working-directory: ./bindings/python
+        run: |
+          MACOSX_DEPLOYMENT_TARGET=10.11 python setup.py bdist_wheel
 
       - name: Upload wheels
         shell: bash
         run: |
           pip install awscli
           aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR"
 
-  create_wheels_macos_13:
-    name: MacOS 13
-    runs-on: ${{ matrix.os }}
+  create_wheels_macos:
+    name: MacOS
+    runs-on: ${{ matrix.os.os }}
+    needs: [lock_exists]
     strategy:
       matrix:
-        os: [macos-13]
         python: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        os: [{os: "macos-11", target: "11.0"}, {os: "macos-12"}, {os: "macos-13"}, {os: "macos-13", target: "14.0"}]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v2
@@ -136,35 +163,36 @@ jobs:
         uses: actions/setup-python@v4
         with:
           python-version: ${{ matrix.python }}
-          architecture: x64
 
       - name: Install dependencies
         run: |
           # On old versions of python there is an old version of setuptools already installed
-          pip install setuptools wheel setuptools-rust==0.11.3 --ignore-installed --force-reinstall
+          pip install setuptools wheel setuptools-rust --ignore-installed --force-reinstall
 
-      - name: Build wheel
+      - name: Override target
+        if: ${{ matrix.os.target }}
         working-directory: ./bindings/python
-        run: python setup.py bdist_wheel
+        run: echo "MACOSX_DEPLOYMENT_TARGET=${{ matrix.os.target }}" >> $GITHUB_ENV
 
-      - name: Rename wheels
-        shell: bash
-        working-directory: ./bindings/python/dist
-        run: for file in *.whl ; do mv $file ${file//macosx_10_1[0-9]/macosx_10_11} || true; done
+      - name: Build wheel
+        working-directory: ./bindings/python
+        run: echo $MACOX_DEPLOYMENT_TARGET && python setup.py bdist_wheel
 
       - name: Upload wheels
         shell: bash
         run: |
           pip install awscli
           aws s3 sync --exact-timestamps ./bindings/python/dist "s3://safetensors-releases/python/$DIST_DIR"
 
+
   create_wheels_macos_arm64:
-    name: Create wheels for MacOS M1
+    name: MacOS M1
     runs-on: macos-arm64
+    needs: [lock_exists]
     strategy:
       matrix:
         python: ["3.9.13", "3.10.6", "3.11.0"]
-        target: ["12.0", "13.0"]
+        target: ["12.0", "13.0", "14.0"]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
@@ -194,7 +222,7 @@ jobs:
   Upload_package:
     name: Upload package to PyPi
     runs-on: ubuntu-latest
-    needs: [create_wheels_manylinux, create_wheels_windows_32bit, create_wheels_others_64bit, create_wheels_macos_arm64, create_wheels_macos_13]
+    needs: [create_wheels_manylinux, create_wheels_windows, create_wheels_macos, create_wheels_macos_arm64, create_wheels_macos_conda]
 
     steps:
       - uses: actions/checkout@v3