Enable Python Tests on Linux CI and Add a Nightly Pipeline (#136)

.github/workflows/linux-cpu-arm64-build.yml

@@ -42,12 +42,3 @@ jobs:
             --container-registry onnxruntimebuildcache \
             --repository onnxruntimecpubuild
           docker run --rm --volume $GITHUB_WORKSPACE:/onnxruntime_src  -w /onnxruntime_src onnxruntimecpubuild bash -c "echo $PATH && /usr/bin/cmake --preset linux_gcc_cpu_release && /usr/bin/cmake --build --preset linux_gcc_cpu_release"
-
-      # TODO: Re-enable these tests when python version is updated
-      # - name: Install the onnxruntime-genai Python wheel and run Python tests
-      #   run: |
-      #     echo "Installing the onnxruntime-genai Python wheel and running the Python tests"
-      #     docker run \
-      #       --rm \
-      #       --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-      #       -w /onnxruntime_src onnxruntimecpubuild bash -c "python3 -m pip install /onnxruntime_src/build/gcc_cpu/release/wheel/onnxruntime_genai*.whl && python3 -m pip install -r test/python/requirements.txt && python3 test/python/test_onnxruntime_genai.py --cwd test/python --test_models test/test_models"

.github/workflows/linux-cpu-x64-build.yml

@@ -47,15 +47,14 @@ jobs:
           cmake --preset linux_clang_cpu_release
           cmake --build --preset linux_clang_cpu_release
 
-      # TODO: Reenable tests once permission denied error goes away.
-      # - name: Install the python wheel and test dependencies
-      #   run: |
-      #     python3 -m pip install build/clang_cpu/release/wheel/onnxruntime_genai*.whl
-      #     python3 -m pip install -r test/python/requirements.txt
+      - name: Install the python wheel and test dependencies
+        run: |
+          python3 -m pip install build/clang_cpu/release/wheel/onnxruntime_genai*.whl
+          python3 -m pip install -r test/python/requirements.txt --user
 
-      # - name: Run the python tests
-      #   run: |
-      #     python3 test/python/test_onnxruntime_genai.py --cwd test/python --test_models test/test_models
+      - name: Run the python tests
+        run: |
+          python3 test/python/test_onnxruntime_genai.py --cwd test/python --test_models test/test_models
 
       - name: Verify Build Artifacts
         if: always()

.github/workflows/linux-cpu-x64-nightly-build.yml

@@ -0,0 +1,76 @@
+name: "Linux CPU x64 Nightly Build"
+
+# Run at 5:00 AM UTC every day
+# 9:00 PM PST
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 5 * * *"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+env:
+  ort_dir: "onnxruntime-linux-x64-1.17.0"
+  ort_zip: "onnxruntime-linux-x64-1.17.0.tgz"
+  ort_url: "https://github.com/microsoft/onnxruntime/releases/download/v1.17.0/onnxruntime-linux-x64-1.17.0.tgz"
+jobs:
+  job:
+    runs-on: [ "self-hosted", "1ES.Pool=onnxruntime-genai-Ubuntu2204-AMD-CPU" ]
+    steps:
+      - name: Checkout OnnxRuntime GenAI repo
+        uses: actions/checkout@v2
+
+
+
+      - name: Download OnnxRuntime
+        run: |
+          curl -L -o ${{ env.ort_zip }} ${{ env.ort_url }} 
+
+      - name: Unzip OnnxRuntime
+        run: |
+          tar -xzf ${{ env.ort_zip }}
+          rm ${{ env.ort_zip }}
+
+      - name: Rename OnnxRuntime to ort
+        run: |
+          mv ${{ env.ort_dir }} ort
+
+      - name: Git Submodule Update
+        run: |
+          git submodule update --init --recursive
+          
+      - name: Build with CMake and clang
+        run: |
+          set -e -x
+          rm -rf build
+          cmake --preset linux_clang_cpu_release
+          cmake --build --preset linux_clang_cpu_release
+
+      - name: Install the python wheel and test dependencies
+        run: |
+          python3 -m pip install build/clang_cpu/release/wheel/onnxruntime_genai*.whl
+          python3 -m pip install -r test/python/requirements-nightly-cpu.txt --user
+
+      - name: Get HuggingFace Token
+        run: |
+          az login --identity --username 63b63039-6328-442f-954b-5a64d124e5b4
+          HF_TOKEN=$(az keyvault secret show --vault-name anubissvcsecret --name ANUBIS-HUGGINGFACE-TOKEN --query value)
+          echo "::add-mask::$HF_TOKEN"
+          echo "HF_TOKEN=$HF_TOKEN" >> $GITHUB_ENV
+
+      - name: Run the python tests
+        run: |
+          python3 test/python/test_onnxruntime_genai.py --cwd test/python --test_models test/test_models --e2e
+
+      - name: Verify Build Artifacts
+        if: always()
+        run: |
+          ls -l ${{ github.workspace }}/build
+
+      - name: Upload Build Artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: onnxruntime-genai-linux-cpu-x64
+          path: ${{ github.workspace }}/build/**/*.a

.github/workflows/linux-gpu-x64-build.yml

@@ -56,12 +56,11 @@ jobs:
             --volume $GITHUB_WORKSPACE:/onnxruntime_src \
             -w /onnxruntime_src onnxruntimegpubuild bash -c "echo $PATH && /usr/bin/cmake -DCMAKE_CUDA_ARCHITECTURES=86 --preset linux_gcc_cuda_release && /usr/bin/cmake --build --preset linux_gcc_cuda_release"
 
-      # TODO: Re-enable these tests when python version is updated
-      # - name: Install the onnxruntime-genai Python wheel and run Python tests
-      #   run: |
-      #     echo "Installing the onnxruntime-genai Python wheel and running the Python tests"
-      #     docker run \
-      #       --gpus all \
-      #       --rm \
-      #       --volume $GITHUB_WORKSPACE:/onnxruntime_src \
-      #       -w /onnxruntime_src onnxruntimegpubuild bash -c "python3 -m pip install /onnxruntime_src/build/gcc_cuda/release/wheel/onnxruntime_genai*.whl && python3 -m pip install -r test/python/requirements.txt && python3 test/python/test_onnxruntime_genai.py --cwd test/python --test_models test/test_models"
+      - name: Install the onnxruntime-genai Python wheel and run Python tests
+        run: |
+          echo "Installing the onnxruntime-genai Python wheel and running the Python tests"
+          docker run \
+            --gpus all \
+            --rm \
+            --volume $GITHUB_WORKSPACE:/onnxruntime_src \
+            -w /onnxruntime_src onnxruntimegpubuild bash -c "python3 -m pip install /onnxruntime_src/build/gcc_cuda/release/wheel/onnxruntime_genai*.whl --user && python3 -m pip install -r test/python/requirements.txt --user && python3 test/python/test_onnxruntime_genai.py --cwd test/python --test_models test/test_models"

test/python/_test_utils.py

@@ -0,0 +1,52 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import logging
+import os
+import subprocess
+import sys
+from typing import Dict, List, Optional, Union
+
+
+def is_windows():
+    return sys.platform.startswith("win")
+
+
+def run_subprocess(
+    args: List[str],
+    cwd: Optional[Union[str, bytes, os.PathLike]] = None,
+    capture: bool = False,
+    dll_path: Optional[Union[str, bytes, os.PathLike]] = None,
+    shell: bool = False,
+    env: Dict[str, str] = {},
+    log: Optional[logging.Logger] = None,
+):
+    if log:
+        log.info(f"Running subprocess in '{cwd or os.getcwd()}'\n{args}")
+    user_env = os.environ.copy()
+    user_env.update(env)
+    if dll_path:
+        if is_windows():
+            user_env["PATH"] = dll_path + os.pathsep + user_env["PATH"]
+        else:
+            if "LD_LIBRARY_PATH" in user_env:
+                user_env["LD_LIBRARY_PATH"] += os.pathsep + dll_path
+            else:
+                user_env["LD_LIBRARY_PATH"] = dll_path
+
+    stdout, stderr = (subprocess.PIPE, subprocess.STDOUT) if capture else (None, None)
+    completed_process = subprocess.run(
+        args,
+        cwd=cwd,
+        check=True,
+        stdout=stdout,
+        stderr=stderr,
+        env=user_env,
+        shell=shell,
+    )
+
+    if log:
+        log.debug(
+            "Subprocess completed. Return code=" + str(completed_process.returncode)
+        )
+    return completed_process

test/python/conftest.py

@@ -5,7 +5,12 @@
 
 
 def pytest_addoption(parser):
-    parser.addoption("--test_models", help="Path to the current working directory", type=str, required=True)
+    parser.addoption(
+        "--test_models",
+        help="Path to the current working directory",
+        type=str,
+        required=True,
+    )
 
 
 @pytest.fixture

test/python/requirements-nightly-cpu.txt

@@ -0,0 +1,8 @@
+-f https://download.pytorch.org/whl/torch_stable.html
+torch==2.2.1+cpu
+numpy
+pytest
+onnx
+onnxruntime_gpu
+transformers
+huggingface_hub[cli]

test/python/test_onnxruntime_genai.py

@@ -1,80 +1,69 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
-from __future__ import annotations
-
 import argparse
 import logging
 import os
-import subprocess
+import pathlib
 import sys
+from typing import Union
+
+from _test_utils import run_subprocess
 
 logging.basicConfig(
     format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG
 )
 log = logging.getLogger("onnxruntime-genai-tests")
 
 
-def is_windows():
-    return sys.platform.startswith("win")
-
-
-def run_subprocess(
-    args: list[str],
-    cwd: str | bytes | os.PathLike | None = None,
-    capture: bool = False,
-    dll_path: str | bytes | os.PathLike | None = None,
-    shell: bool = False,
-    env: dict[str, str] = {},
-    log: logging.Logger | None = None,
+def run_onnxruntime_genai_api_tests(
+    cwd: Union[str, bytes, os.PathLike],
+    log: logging.Logger,
+    test_models: Union[str, bytes, os.PathLike],
 ):
-    if log:
-        log.info(f"Running subprocess in '{cwd or os.getcwd()}'\n{args}")
-    user_env = os.environ.copy()
-    user_env.update(env)
-    if dll_path:
-        if is_windows():
-            user_env["PATH"] = dll_path + os.pathsep + user_env["PATH"]
-        else:
-            if "LD_LIBRARY_PATH" in user_env:
-                user_env["LD_LIBRARY_PATH"] += os.pathsep + dll_path
-            else:
-                user_env["LD_LIBRARY_PATH"] = dll_path
-
-    stdout, stderr = (subprocess.PIPE, subprocess.STDOUT) if capture else (None, None)
-    completed_process = subprocess.run(
-        args,
-        cwd=cwd,
-        check=True,
-        stdout=stdout,
-        stderr=stderr,
-        env=user_env,
-        shell=shell,
-    )
+    log.debug("Running: ONNX Runtime GenAI API Tests")
 
-    if log:
-        log.debug(
-            "Subprocess completed. Return code=" + str(completed_process.returncode)
-        )
-    return completed_process
+    command = [
+        sys.executable,
+        "-m",
+        "pytest",
+        "-sv",
+        "test_onnxruntime_genai_api.py",
+        "--test_models",
+        test_models,
+    ]
 
+    run_subprocess(command, cwd=cwd, log=log).check_returncode()
 
-def run_onnxruntime_genai_api_tests(cwd: str | bytes | os.PathLike, log: logging.Logger, 
-                                    test_models: str | bytes | os.PathLike):
-    log.debug("Running: ONNX Runtime GenAI API Tests")
 
-    command = [sys.executable, "-m", "pytest", "-sv", "test_onnxruntime_genai_api.py", "--test_models", test_models]
+def run_onnxruntime_genai_e2e_tests(
+    cwd: Union[str, bytes, os.PathLike],
+    log: logging.Logger,
+):
+    log.debug("Running: ONNX Runtime GenAI E2E Tests")
 
+    log.debug("Running: Phi-2")
+    command = [sys.executable, "test_onnxruntime_genai_phi2.py"]
     run_subprocess(command, cwd=cwd, log=log).check_returncode()
 
 
 def parse_arguments():
     parser = argparse.ArgumentParser()
-    parser.add_argument("--cwd", help="Path to the current working directory")
+    parser.add_argument(
+        "--cwd",
+        help="Path to the current working directory",
+        default=pathlib.Path(__file__).parent.resolve().absolute(),
+    )
     parser.add_argument(
         "--test_models",
         help="Path to the test_models directory",
-        required=True
+        default=pathlib.Path(__file__).parent.parent.resolve().absolute()
+        / "test_models",
+    )
+    parser.add_argument(
+        "--e2e",
+        help="Whether to run e2e tests. If not specified e2e tests will not run.",
+        action="store_true",
     )
     return parser.parse_args()
 
@@ -84,7 +73,12 @@ def main():
 
     log.info("Running onnxruntime-genai tests pipeline")
 
-    run_onnxruntime_genai_api_tests(os.path.abspath(args.cwd), log, os.path.abspath(args.test_models))
+    run_onnxruntime_genai_api_tests(
+        os.path.abspath(args.cwd), log, os.path.abspath(args.test_models)
+    )
+
+    if args.e2e:
+        run_onnxruntime_genai_e2e_tests(os.path.abspath(args.cwd), log)
 
     return 0
 

test/python/test_onnxruntime_genai_api.py

@@ -29,7 +29,7 @@ def test_greedy_search(device, test_data_path, relative_model_path):
     search_params.input_ids = np.array(
         [[0, 0, 0, 52], [0, 0, 195, 731]], dtype=np.int32
     )
-    search_params.set_search_options({"max_length":10})
+    search_params.set_search_options({"max_length": 10})
     input_ids_shape = [2, 4]
     batch_size = input_ids_shape[0]
 
@@ -132,7 +132,7 @@ def test_batching(device, test_data_path, relative_model_path):
     ]
 
     params = og.GeneratorParams(model)
-    params.set_search_options({"max_length":20}) # To run faster
+    params.set_search_options({"max_length": 20})  # To run faster
     params.set_input_sequences(tokenizer.encode_batch(prompts))
 
     output_sequences = model.generate(params)