diff --git a/Jenkinsfile b/Jenkinsfile index bf29cf2b2..4675d68af 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,10 +37,8 @@ pipeline { stage("Setup") { steps { println "Stage running on: ${env.NODE_NAME}" - // clone checkout scm - sh 'git submodule update --init --recursive --jobs \$(nproc)' - // create venv and install pip packages + sh "./build.sh -T init" createVenv("requirements.txt") withVenv { sh "pip install -r requirements.txt" @@ -50,11 +48,8 @@ pipeline { stage("Build") { steps { withVenv { - // apply tflite-micro patch - dir("third_party/lib_tflite_micro") { - sh "make patch" - } // build dll_interpreter for python interface + sh "./build.sh -T runtime-host -b" sh "./build.sh -T xinterpreter-nozip -b" // build xformer dir("xformer") { @@ -84,12 +79,12 @@ pipeline { steps { withVenv { dir("xformer") { - // xformer2 unit tests sh "./bazelisk-linux-amd64 --output_user_root=${env.BAZEL_USER_ROOT} test --remote_cache=${env.BAZEL_CACHE_URL} //Test:all --verbose_failures --test_output=errors --//:disable_version_check" } - // xformer2 integration tests sh "pytest integration_tests/runner.py --models_path integration_tests/models/non-bnns -n 8 --junitxml=integration_tests/integration_non_bnns_junit.xml" sh "pytest integration_tests/runner.py --models_path integration_tests/models/bnns --bnn -n 8 --junitxml=integration_tests/integration_bnns_junit.xml" + sh "pytest integration_tests/runner.py --models_path integration_tests/models/non-bnns --compiled -n 8 --junitxml=integration_compiled_non_bnns_junit.xml" + sh "pytest integration_tests/runner.py --models_path integration_tests/models/bnns --bnn --compiled -n 8 --junitxml=integration_compiled_bnns_junit.xml" // Any call to pytest can be given the "--junitxml SOMETHING_junit.xml" option // This step collects these files for display in Jenkins UI junit "**/*_junit.xml" diff --git a/build.sh b/build.sh index 06414322b..c212b7230 100755 --- a/build.sh +++ b/build.sh @@ -118,13 +118,17 @@ create_zip() { cd third_party/lib_tflite_micro mkdir -p build cd build - cmake .. --toolchain=../lib_tflite_micro/submodules/xmos_cmake_toolchain/xs3a.cmake + if [ "$1" = "xcore" ]; then + cmake .. --toolchain=../lib_tflite_micro/submodules/xmos_cmake_toolchain/xs3a.cmake + else + cmake .. -DLIB_NAME=x86tflitemicro + fi make create_zip -j$NUM_PROCS cd $SCRIPT_DIR mv third_party/lib_tflite_micro/build/release_archive.zip python/xmos_ai_tools/runtime/release_archive.zip cd python/xmos_ai_tools/runtime - rm -rf lib include - unzip release_archive.zip + rm -rf include + unzip -o release_archive.zip rm release_archive.zip cd $SCRIPT_DIR } @@ -147,6 +151,10 @@ clean_xinterpreter() { make -C python/xmos_ai_tools/xinterpreters/host clean } +clean_runtime() { + rm -rf third_party/lib_tflite_micro/build +} + test_xinterpreter() { echo "Not implemented yet" exit 1 @@ -161,24 +169,18 @@ case $TARGET in patch ;; xformer) - case $ACTION in - --build) - build_xformer - ;; - *) - unsupported_action - ;; - esac + build_xformer ;; xinterpreter) case $ACTION in --build) version_check - create_zip + create_zip "xcore" build_xinterpreter ;; --clean) clean_xinterpreter + clean_runtime ;; --test) test_xinterpreter @@ -189,23 +191,42 @@ case $TARGET in esac ;; # this is a mess: xinterpreter-nozip only used for CI - xinterpreter-nozip) + runtime-host) case $ACTION in --build) version_check - build_xinterpreter + clean_runtime + create_zip "x86" + clean_runtime + ;; + --clean) + clean_runtime ;; *) unsupported_action ;; esac ;; + xinterpreter-nozip) + case $ACTION in + --build) + version_check + build_xinterpreter + ;; + --clean) + clean_xinterpreter + ;; + --test) + test_xinterpreter + ;; + esac + ;; all) case $ACTION in --build) version_check build_xformer - create_zip + create_zip "xcore" build_xinterpreter ;; --clean) diff --git a/integration_tests/runner.py b/integration_tests/runner.py index 966e182d4..cdc73c914 100644 --- a/integration_tests/runner.py +++ b/integration_tests/runner.py @@ -12,6 +12,7 @@ import tensorflow as tf from xmos_ai_tools.xinterpreters import xcore_tflm_host_interpreter from xmos_ai_tools import xformer +import xmos_ai_tools.runtime as rt import yaml from abc import ABC, abstractmethod, abstractproperty @@ -72,9 +73,16 @@ def __init__(self, model): self._interpreter.set_model(model_content=model, secondary_memory=False) self._dets = self._interpreter.get_output_details() + # Try/except in case we cancel operation before interpreter/dir initialised def __del__(self): - self._interpreter.close() - self._temp_dir.cleanup() + try: + self._interpreter.close() + except AttributeError: + pass + try: + self._temp_dir.cleanup() + except AttributeError: + pass class BnnInterpreter(AbstractRefRunner): @@ -124,10 +132,7 @@ def input_details(self): class XFHostRuntime(AbstractXFRunner): def __init__(self, model_content): - path_var = os.getenv("XMOS_AITOOLSLIB_PATH") - if path_var is None: - LOGGER.error("XMOS_AITOOLSLIB_PATH not set properly.") - sys.exit(1) + path_var = os.path.dirname(rt.__file__) super().__init__(model_content) self._model_exe_path = self._dir_path / "a.out" cmd = [ @@ -145,7 +150,7 @@ def __init__(self, model_content): f"{MAIN_CPP_PATH}", "-o", f"{self._model_exe_path}", - "-lxtflitemicro", + "-lx86tflitemicro", ] print(" ".join(cmd)) run_cmd(cmd) diff --git a/third_party/lib_tflite_micro b/third_party/lib_tflite_micro index d4d002e6d..37310d212 160000 --- a/third_party/lib_tflite_micro +++ b/third_party/lib_tflite_micro @@ -1 +1 @@ -Subproject commit d4d002e6d6047bfc1a18e1ec0de2c157b354d50e +Subproject commit 37310d21265d43fe6d838f8c5bae70e6caedd9a5