Validate ExecuTorch path for more models in CI

pytorch · Apr 9, 2024 · eb57f3f · eb57f3f
1 parent 67cad1f
commit eb57f3f
Show file tree

Hide file tree

Showing 4 changed files with 228 additions and 26 deletions.
diff --git a/.ci/scripts/download_checkpoints.sh b/.ci/scripts/download_checkpoints.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -exu
+
+CHECKPOINT_NAME="$1"
+RESOURCES_STRING="$2"
+
+pushd "${LLAMA_FAST_ROOT}" || exit
+
+# Create the directory for the checkpoint
+mkdir -p "checkpoints/${CHECKPOINT_NAME}"
+cd "checkpoints/${CHECKPOINT_NAME}" || exit
+
+# Download all resources
+IFS=',' # Set the field separator to comma
+for resource in $RESOURCES_STRING; do
+  echo "Downloading: $resource"
+  if ! wget "$resource" 2>&1; then
+    echo "Error: Failed to download $resource" >&2
+    exit 1
+  fi
+done
+
+popd || exit
diff --git a/.github/workflows/executorch.yml b/.github/workflows/executorch.yml
@@ -0,0 +1,71 @@
+name: executorch
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  gather-models:
+    runs-on: ubuntu-22.04
+    outputs:
+      models: ${{ steps.gather-models.outputs.models }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: 'false'
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Extract the list of models to test
+        id: gather-models
+        run: |
+          set -eux
+          PYTHONPATH="${PWD}" python ./scripts/gather_test_models.py
+  test-models-executorch:
+    name: test-models-executorch (${{ matrix.checkpoint_name }}, ${{ matrix.runner }})
+    needs: gather-models
+    strategy:
+      matrix: ${{ fromJSON(needs.gather-models.outputs.models) }}
+      fail-fast: false
+    runs-on: ${{ matrix.runner }}
+    env:
+      LLAMA_FAST_ROOT: ${{ github.workspace }}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v3
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+      - name: Print machine info
+        run: |
+          echo "Platform: $(uname -s)"
+          echo "Processor type: $(uname -p)"
+          echo "Number of processors: $(nproc)"
+          echo "RAM: $(free -h | awk '/Mem/ {print $2}')"
+          echo "Disk space: $(df -h --total | awk '/total/ {print $2}')"
+      - name: Install ExecuTorch
+        run: |
+          bash ${LLAMA_FAST_ROOT}/scripts/install_et.sh
+      - name: Download checkpoints
+        run: |
+          bash ${LLAMA_FAST_ROOT}/.ci/scripts/download_checkpoints.sh ${{ matrix.checkpoint_name }} "${{ matrix.resources }}"
+      - name: Run inferencec
+        run: |
+          set -exu
+          pushd ${LLAMA_FAST_ROOT}
+          export CHECKPOINT_NAME=${{ matrix.checkpoint_name }}
+          export CHECKPOINT_PATH=${LLAMA_FAST_ROOT}/checkpoints/${CHECKPOINT_NAME}/${CHECKPOINT_NAME}.pt
+
+          python generate.py --checkpoint-path ${CHECKPOINT_PATH} --temperature 0 > ${PWD}/output_eager
+          cat ${PWD}/output_eager
+
+          python export.py --checkpoint-path ${CHECKPOINT_PATH} --output-pte-path ${PWD}/${CHECKPOINT_NAME}.pte
+
+          python generate.py --checkpoint-path ${CHECKPOINT_PATH} --temperature 0 --pte-path ${PWD}/${CHECKPOINT_NAME}.pte  > ${PWD}/output_et
+          cat ${PWD}/output_et
+
+          echo "Tests complete."
diff --git a/scripts/gather_test_models.py b/scripts/gather_test_models.py
@@ -0,0 +1,75 @@
+#!/usr/bin/env python
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+import itertools
+import json
+import os
+from typing import Any
+
+
+MODEL_CHECKPOINT_NAMES = {
+    "stories15M": "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt,https://github.com/karpathy/llama2.c/raw/master/tokenizer.model,https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin",
+    "stories42M": "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories42M.pt,https://github.com/karpathy/llama2.c/raw/master/tokenizer.model,https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin",
+    "stories110M": "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt,https://github.com/karpathy/llama2.c/raw/master/tokenizer.model,https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin",
+    # "open_llama_7b",
+    # "Llama-2-7b-chat-hf",
+    # "Llama-2-13b-chat-hf",
+    # "Llama-2-70b-chat-hf",
+    "CodeLlama-7b-Python-hf": "https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/config.json,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/generation_config.json,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/model-00001-of-00002.safetensors,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/model-00002-of-00002.safetensors,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/model.safetensors.index.json,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/pytorch_model-00001-of-00003.bin,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/pytorch_model-00002-of-00003.bin,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/pytorch_model-00003-of-00003.bin,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/pytorch_model.bin.index.json,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/special_tokens_map.json,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/tokenizer.json,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/tokenizer.model,https://huggingface.co/codellama/CodeLlama-7b-Python-hf/blob/main/tokenizer_config.json",
+    # "CodeLlama-34b-Python-hf",
+    "Mistral-7B-v0.1": "https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/config.json,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/generation_config.json,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/model-00001-of-00002.safetensors,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/model-00002-of-00002.safetensors,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/model.safetensors.index.json,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/pytorch_model-00001-of-00002.bin,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/pytorch_model-00002-of-00002.bin,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/pytorch_model.bin.index.json,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/special_tokens_map.json,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/tokenizer.json,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/tokenizer.model,https://huggingface.co/mistralai/Mistral-7B-v0.1/blob/main/tokenizer_config.json"
+    # "Mistral-7B-Instruct-v0.1",
+    # "Mistral-7B-Instruct-v0.2",
+}
+
+JOB_RUNNERS = [
+    "32-core-ubuntu",
+    "macos-12",
+    "macos-14",
+]
+
+
+def set_output(name: str, val: Any) -> None:
+    """
+    Set the GitHb output so that it can be accessed by other jobs
+    """
+    print(f"Setting {val} to GitHub output")
+
+    if os.getenv("GITHUB_OUTPUT"):
+        with open(str(os.getenv("GITHUB_OUTPUT")), "a") as env:
+            print(f"{name}={val}", file=env)
+    else:
+        print(f"::set-output name={name}::{val}")
+
+
+def export_models_for_ci() -> dict[str, dict]:
+    """
+    This gathers all the models that we want to test on GitHub OSS CI
+    """
+
+    # This is the JSON syntax for configuration matrix used by GitHub
+    # https://docs.github.com/en/actions/using-jobs/using-a-matrix-for-your-jobs
+    models = {"include": []}
+
+    for checkpoint_name, runner in itertools.product(
+        MODEL_CHECKPOINT_NAMES,
+        JOB_RUNNERS,
+    ):
+        record = {
+            "checkpoint_name": checkpoint_name,
+            "resources": MODEL_CHECKPOINT_NAMES[checkpoint_name],
+            "runner": runner,
+            "timeout": 90,
+        }
+
+        models["include"].append(record)
+
+    set_output("models", json.dumps(models))
+
+
+if __name__ == "__main__":
+    export_models_for_ci()
diff --git a/scripts/install_et.sh b/scripts/install_et.sh
@@ -1,26 +1,53 @@
-cd ${LLAMA_FAST_ROOT}
-echo "Inside: $LLAMA_FAST_ROOT"
-
-echo "Cloning executorch to ${LLAMA_FAST_ROOT}/build/src"
-rm -rf ${LLAMA_FAST_ROOT}/build
-mkdir -p ${LLAMA_FAST_ROOT}/build/src
-cd ${LLAMA_FAST_ROOT}/build/src
-git clone https://github.com/pytorch/executorch.git
-cd executorch
-echo "Install executorch: submodule update"
-git submodule sync
-git submodule update --init
-
-echo "Applying fixes"
-cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/module.cpp ${LLAMA_FAST_ROOT}/build/src/executorch/extension/module/module.cpp # ET uses non-standard C++ that does not compile in GCC
-cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/managed_tensor.h ${LLAMA_FAST_ROOT}/build/src/executorch/extension/runner_util/managed_tensor.h # ET is missing headers for vector/memory.  This causes downstream issues when building runner-et.
-
-echo "Building and installing python libraries"
-./install_requirements.sh --pybind xnnpack
-
-echo "Building and installing C++ libraries"
-echo "Inside: ${PWD}"
-mkdir cmake-out
-cmake -DCMAKE_BUILD_TYPE=Release -DEXECUTORCH_BUILD_OPTIMIZED=ON -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON -DEXECUTORCH_BUILD_XNNPACK=ON -S . -B cmake-out -G Ninja
-cmake --build cmake-out
-cmake --install cmake-out --prefix ${LLAMA_FAST_ROOT}/build/install
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -exu
+
+install_pip_dependencies() {
+  echo "Intalling common pip packages"
+
+  pip install wheel
+  pip install cmake
+  pip install ninja
+  pip install zstd
+  pushd ${LLAMA_FAST_ROOT}
+  pip install -r ./requirements.txt
+  popd
+}
+
+install_executorch() {
+  echo "Cloning executorch to ${LLAMA_FAST_ROOT}/build/src"
+  rm -rf ${LLAMA_FAST_ROOT}/build
+  mkdir -p ${LLAMA_FAST_ROOT}/build/src
+  pushd ${LLAMA_FAST_ROOT}/build/src
+  git clone https://github.com/pytorch/executorch.git
+  cd executorch
+  echo "Install executorch: submodule update"
+  git submodule sync
+  git submodule update --init
+
+  echo "Applying fixes"
+  cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/module.cpp ${LLAMA_FAST_ROOT}/build/src/executorch/extension/module/module.cpp # ET uses non-standard C++ that does not compile in GCC
+  cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/managed_tensor.h ${LLAMA_FAST_ROOT}/build/src/executorch/extension/runner_util/managed_tensor.h # ET is missing headers for vector/memory.  This causes downstream issues when building runner-et.
+
+  echo "Building and installing python libraries"
+  bash ./install_requirements.sh --pybind xnnpack
+  pip list
+
+  echo "Building and installing C++ libraries"
+  echo "Inside: ${PWD}"
+  mkdir cmake-out
+  cmake -DCMAKE_BUILD_TYPE=Release -DEXECUTORCH_BUILD_OPTIMIZED=ON -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON -DEXECUTORCH_BUILD_XNNPACK=ON -S . -B cmake-out -G Ninja
+  cmake --build cmake-out
+  cmake --install cmake-out --prefix ${LLAMA_FAST_ROOT}/build/install
+  popd
+}
+
+pushd ${LLAMA_FAST_ROOT}
+install_pip_dependencies
+install_executorch
+popd