diff --git a/.github/workflows/runner_et.yml b/.github/workflows/runner_et.yml
new file mode 100644
index 000000000..da84507fc
--- /dev/null
+++ b/.github/workflows/runner_et.yml
@@ -0,0 +1,65 @@
+name: Compile main
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  run-tinystories:
+    strategy:
+      matrix:
+        runner: [macos-14-xlarge]
+    runs-on: ${{matrix.runner}}
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.11
+      - name: Print machine info
+        run: |
+          uname -a
+          if [ $(uname -s) == Darwin ]; then
+            sysctl machdep.cpu.brand_string
+            sysctl machdep.cpu.core_count
+          fi
+      - name: Install requirements
+        run: |
+          echo "Intalling pip packages"
+          pip install wheel
+          pip install cmake
+          pip install ninja
+          pip install zstd
+          pip install -r requirements.txt
+
+          export LLAMA_FAST_ROOT=${PWD}
+          export ET_NO_PYBIND=1
+          ./scripts/install_et.sh
+          cmake -S ./runner-et -B build/cmake-out -G Ninja
+          cmake --build ./build/cmake-out
+      - name: Download checkpoints
+        run: |
+          mkdir -p checkpoints/stories15M
+          pushd checkpoints/stories15M
+          wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.pt
+          wget https://github.com/karpathy/llama2.c/raw/master/tokenizer.model
+          wget https://github.com/karpathy/llama2.c/raw/master/tokenizer.bin
+          popd
+      - name: Run inference
+        run: |
+          export MODEL_DIR=${PWD}/checkpoints/stories15M
+          export PROMPT="Once upon a time in a land far away"
+
+          python generate.py --checkpoint-path ${MODEL_DIR}/stories15M.pt --temperature 0 --prompt "${PROMPT}"  > ${PWD}/output_eager
+          cat ${PWD}/output_eager
+
+          python export.py --checkpoint-path ${MODEL_DIR}/stories15M.pt --output-pte-path ${PWD}/stories15M.pte
+
+          ./build/cmake-out/runner_et ${PWD}/stories15M.pte -z ${MODEL_DIR}/tokenizer.bin -i "${PROMPT}" > ${PWD}/output_et
+          cat ${PWD}/output_et
+
+          echo "Tests complete."
diff --git a/scripts/install_et.sh b/scripts/install_et.sh
index dbf79a49a..a40c07f01 100755
--- a/scripts/install_et.sh
+++ b/scripts/install_et.sh
@@ -16,7 +16,13 @@ cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/module.cpp ${LLAMA_FAST_ROOT}/build/src/e
 cp ${LLAMA_FAST_ROOT}/scripts/fixes_et/managed_tensor.h ${LLAMA_FAST_ROOT}/build/src/executorch/extension/runner_util/managed_tensor.h # ET is missing headers for vector/memory.  This causes downstream issues when building runner-et.
 
 echo "Building and installing python libraries"
-./install_requirements.sh --pybind xnnpack
+if [ -n "${ET_NO_PYBIND}" ]; then
+    echo "Not installing pybind"
+    ./install_requirements.sh
+else
+    echo "Installing pybind"
+    ./install_requirements.sh --pybind xnnpack
+fi
 
 echo "Building and installing C++ libraries"
 echo "Inside: ${PWD}"