Merge branch 'main' of https://github.com/talmolab/sleap-deps into el…

…izabeth/add-windows-linux-tensorflow-conda-package

.conda.tensorflow_macos/build.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+export PIP_NO_INDEX=False
+export PIP_NO_DEPENDENCIES=False
+export PIP_IGNORE_INSTALLED=False
+
+pip install --no-cache-dir -r ./requirements.tensorflow_macos.txt

.conda.tensorflow_macos/meta.yaml

@@ -0,0 +1,101 @@
+# Ref: https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html
+
+package:
+  name: tensorflow
+  version: 2.12.0
+
+about:
+  home: https://tensorflow.org
+  license: Apache License
+  summary: "TensorFlow conda package based on the PyPI wheels for Apple Silicon Macs."
+
+build:
+  number: 4
+
+source:
+  # can't find requirements.tensorflow_macos.txt without this
+  path: ../
+
+requirements:
+  host:
+    - conda-forge::absl-py >=1.0.0
+    - conda-forge::astunparse >=1.6.0
+    - conda-forge::cachetools <6.0,>=2.0.0
+    - conda-forge::certifi >=2017.4.17
+    - conda-forge::charset-normalizer <4,>=2
+    - conda-forge::gast <=0.4.0,>=0.2.1
+    - conda-forge::google-auth <3,>=1.6.3
+    - conda-forge::google-auth-oauthlib <0.5,>=0.4.1
+    - conda-forge::google-pasta >=0.1.1
+    - conda-forge::grpcio <2.0,>=1.24.3
+    - conda-forge::h5py >=2.9.0
+    - conda-forge::idna <4,>=2.5
+    - conda-forge::keras <2.13.0,>=2.12.0 # Required for tensorflow-macos
+    - conda-forge::keras-preprocessing >=1.1.1
+    - anaconda::markdown >=2.6.8
+    - conda-forge::MarkupSafe >=2.1.1
+    - conda-forge::numpy >=1.16.5,<2.0 # tensorflow compiled with numpy 1.x
+    - conda-forge::oauthlib >=3.0.0
+    - conda-forge::opt-einsum >=2.3.2
+    - conda-forge::packaging
+    - conda-forge::pip
+    - conda-forge::protobuf >3.20 # Needed for pyside6, opencv, and tf compatibility
+    - conda-forge::pyasn1 <0.7.0,>=0.4.6
+    - conda-forge::pyasn1-modules >=0.2.1
+    - conda-forge::python >=3.10.0,<3.11.0
+    - conda-forge::requests <3,>=2.21.0
+    - conda-forge::requests-oauthlib >=0.7.0
+    - conda-forge::rsa <5,>=3.1.4
+    - conda-forge::termcolor >=1.1.0
+    - conda-forge::typing-extensions >=3.6.6
+    - conda-forge::urllib3 <3,>=1.21.1
+    - conda-forge::werkzeug >=1.0.1
+    - conda-forge::wrapt >=1.11.0,<1.15.0 # https://github.com/tensorflow/tensorflow/issues/59869
+
+  run:
+    - conda-forge::absl-py >=1.0.0
+    - conda-forge::astunparse >=1.6.0
+    - conda-forge::cachetools <6.0,>=2.0.0
+    - conda-forge::certifi >=2017.4.17
+    - conda-forge::charset-normalizer <4,>=2
+    - conda-forge::gast <=0.4.0,>=0.2.1
+    - conda-forge::google-auth <3,>=1.6.3
+    - conda-forge::google-auth-oauthlib <0.5,>=0.4.1
+    - conda-forge::google-pasta >=0.1.1
+    - conda-forge::grpcio <2.0,>=1.24.3
+    - conda-forge::h5py >=2.9.0
+    - conda-forge::idna <4,>=2.5
+    - conda-forge::keras >=2.12.0,<2.13.0 # Required for tensorflow-macos==2.12.0
+    - conda-forge::keras-preprocessing >=1.1.1
+    - anaconda::markdown >=2.6.8
+    - conda-forge::MarkupSafe >=2.1.1
+    - conda-forge::numpy >=1.16.5,<2.0 # tensorflow compiled with numpy 1.x
+    - conda-forge::oauthlib >=3.0.0
+    - conda-forge::opt-einsum >=2.3.2
+    - conda-forge::packaging
+    - conda-forge::pip
+    - conda-forge::protobuf >3.20 # Needed for pyside6, opencv, and tf compatibility
+    - conda-forge::pyasn1 <0.7.0,>=0.4.6
+    - conda-forge::pyasn1-modules >=0.2.1
+    - conda-forge::python >=3.10.0,<3.11.0
+    - conda-forge::requests <3,>=2.21.0
+    - conda-forge::requests-oauthlib >=0.7.0
+    - conda-forge::rsa <5,>=3.1.4
+    - conda-forge::termcolor >=1.1.0
+    - conda-forge::typing-extensions >=3.6.6
+    - conda-forge::urllib3 <3,>=1.21.1
+    - conda-forge::werkzeug >=1.0.1
+    - conda-forge::wrapt >=1.11.0,<1.15.0 # https://github.com/tensorflow/tensorflow/issues/59869
+
+    # the following are pip installed, even when conda-forge versions are available
+    # - conda-forge::flatbuffers <2,>=1.12
+    # - conda-forge::libclang >=13.0.0
+    # - conda-forge::six >=1.12.0
+    # - conda-forge::tensorboard <2.10,>=2.9
+    # - conda-forge::tensorboard-data-server <0.7.0,>=0.6.0
+    # - conda-forge::tensorboard-plugin-wit >=1.6.0
+    # - conda-forge::tensorflow-estimator <2.10.0,>=2.9.0rc0
+
+test:
+  imports:
+    - tensorflow

.github/workflows/build_tensorflow.yml

@@ -7,10 +7,12 @@ on:
       - elizabeth/add-windows-linux-tensorflow-conda-package
 
     paths:
-      - '.github/workflows/build_tensorflow.yml'
-      - '.conda.tensorflow/**'
-      - 'environment.tensorflow.yml'
-      - 'requirements.tensorflow.txt'
+      - ".github/workflows/build_tensorflow.yml"
+      - ".conda.tensorflow/**"
+      - ".conda.tensorflow_macos/**"
+      - "environment.tensorflow.yml"
+      - "requirements.tensorflow.txt"
+      - "requirements.tensorflow_macos.txt"
 
 jobs:
   build:
@@ -19,29 +21,29 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ["ubuntu-22.04", "windows-2022"]
+        os: ["ubuntu-22.04", "windows-2022", "macos-14"]
     steps:
       # Checkout the repository
       - name: Checkout
         uses: actions/checkout@v4
 
       - name: Make conda prefix directory (Windows)
-        if: matrix.os == 'windows-2022'
+        if: matrix.os == "windows-2022"
         shell: powershell
         run: |
           New-Item -ItemType Directory -Force -Path C:\c
 
       # https://github.com/conda-incubator/setup-miniconda?tab=readme-ov-file#use-a-different-environment-name-or-path
       - name: Setup Miniconda (Windows)
-        if: matrix.os == 'windows-2022'
+        if: matrix.os == "windows-2022"
         uses: conda-incubator/[email protected]
         with:
           python-version: "3.10"
           environment-file: environment.tensorflow.yml
           activate-environment: C:\c\tf
 
-      - name: Setup Miniconda (Ubuntu)
-        if: matrix.os == 'ubuntu-22.04'
+      - name: Setup Miniconda (not Windows)
+        if: matrix.os != "windows-2022"
         uses: conda-incubator/[email protected]
         with:
           python-version: "3.10"
@@ -56,32 +58,50 @@ jobs:
           conda list
 
       - name: Build conda package (Windows)
-        if: matrix.os == 'windows-2022'
+        if: matrix.os == "windows-2022"
         shell: powershell
         run: |
           conda build --debug .conda.tensorflow --output-folder C:\c\build.tensorflow -c conda-forge
 
       - name: Build conda package (Ubuntu)
-        if: matrix.os == 'ubuntu-22.04'
+        if: matrix.os == "ubuntu-22.04"
         shell: bash -l {0}
         run: |
           conda build --debug .conda.tensorflow --output-folder build.tensorflow -c conda-forge
 
       # https://github.com/anaconda/anaconda-client/issues/529
       - name: Upload to Anaconda (Windows)
-        if: matrix.os == 'windows-2022'
+        if: matrix.os == "windows-2022"
         env:
           ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_SLEAP_DEPS_TOKEN }}
         shell: powershell
         run: |
+          anaconda login --username sleap-deps --password "$ANACONDA_PASSWORD"
           anaconda -v upload "C:\c\build.tensorflow\win-64\*.tar.bz2" --label dev --user sleap-deps
           anaconda logout
 
       - name: Upload to Anaconda (Ubuntu)
-        if: matrix.os == 'ubuntu-22.04'
+        if: matrix.os == "ubuntu-22.04"
         env:
           ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_SLEAP_DEPS_TOKEN }}
         shell: bash -l {0}
         run: |
+          anaconda login --username sleap-deps --password "$ANACONDA_PASSWORD"
           anaconda -v upload build.tensorflow/linux-64/*.tar.bz2 --label dev --user sleap-deps
           anaconda logout
+
+      - name: Build conda package (MacOS)
+        if: matrix.os == "macos-14"
+        shell: bash -l {0}
+        run: |
+          conda build --debug .conda.tensorflow_macos --output-folder build.tensorflow_macos -c conda-forge
+
+      - name: Upload to Anaconda (MacOS)
+        if: matrix.os == "macos-14"
+        env:
+          ANACONDA_PASSWORD: ${{ secrets.ANACONDA_PASSWORD }}
+        shell: bash -l {0}
+        run: |
+          anaconda login --username sleap-deps --password "$ANACONDA_PASSWORD"
+          anaconda -v upload build.tensorflow_macos/osx-arm64/*.tar.bz2 --label dev --user sleap-deps
+          anaconda logout

README.md

@@ -1,2 +1,34 @@
 # sleap-deps
 Packages necessary for sleap build maintained by the SLEAP team.
+
+## tensorflow
+
+There is a Github Actions workflow `build_tensorflow.yml` that uses Github runners to build packages for us (and upload them to the `sleap-deps/label/dev` channel). But, if you would like to locally build and test the tensorflow package:
+
+1. Create the tensorflow build environment:
+
+```bash
+mamba env create -f environment.tensorflow.yml -n tf
+```
+
+2. Activate the tensorflow build environment:
+
+```bash
+mamba activate tf
+```
+
+3. Build the tensorflow conda package:
+
+#### Mac
+
+```bash
+conda build --debug.conda.tensorflow_macos --output-folder build.tensorflow_macos -c conda-forge
+```
+
+4. Test the conda package:
+
+#### Mac
+
+```bash
+mamba create -n tf_macos -c ./build.tensorflow_macos -c conda-forge tensorflow
+```

requirements.tensorflow_macos.txt

@@ -0,0 +1,4 @@
+tensorflow-macos>=2.12.0,<2.13.0; sys_platform == 'darwin' and platform_machine == 'arm64'  # lowest version with protobuf>3.20
+tensorflow-metal; sys_platform == 'darwin' and platform_machine == 'arm64'
+tensorflow-hub; sys_platform == 'darwin' and platform_machine == 'arm64'
+protobuf>3.20; sys_platform == 'darwin' and platform_machine == 'arm64'