Merge branch 'develop' into feature/host_interpreter_tweaks

xmos · Aug 18, 2023 · ca5948a · ca5948a
2 parents 9dd1a79 + f32acdb
commit ca5948a
Show file tree

Hide file tree

Showing 456 changed files with 121 additions and 40,006 deletions.
diff --git a/.github/scripts/build_linux_wheel.sh b/.github/scripts/build_linux_wheel.sh
@@ -27,10 +27,10 @@ CC=/dt9/usr/bin/gcc CXX=/dt9/usr/bin/g++ make build
 # Build xcore-opt
 # Crosstool toolchain info is mentioned here, "--crosstool_top"
 # https://github.com/tensorflow/tensorflow/blob/master/.bazelrc
-cd experimental/xformer
+cd xformer
 bazel build //:xcore-opt --verbose_failures --linkopt=-lrt [email protected]_manylinux2014-cuda11.2-cudnn8.1-tensorrt7.2_config_cuda//crosstool:toolchain --//:disable_version_check
 
 
 # Build python wheel
-cd ../../python
+cd ../python
 python setup.py bdist_wheel
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -58,7 +58,7 @@ jobs:
       - name: Build xformer
         shell: bash
         run: |
-          cd $GITHUB_WORKSPACE/experimental/xformer
+          cd $GITHUB_WORKSPACE/xformer
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
             echo "using Google Cloud Storage cache"
             echo -e 'build --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }}' >> .bazelrc.user
@@ -115,7 +115,7 @@ jobs:
         # auditwheel requires patchelf and repairs the created wheel file to
         # be compatible for manylinux.
         run: |
-          pushd $GITHUB_WORKSPACE/experimental/xformer
+          pushd $GITHUB_WORKSPACE/xformer
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
             echo "using Google Cloud Storage cache"
             echo -e 'build --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }}' >> .bazelrc.user
@@ -199,7 +199,7 @@ jobs:
       - name: Build xformer
         shell: cmd
         run: |
-          cd %GITHUB_WORKSPACE%/experimental/xformer
+          cd %GITHUB_WORKSPACE%/xformer
           IF DEFINED GOOGLE_APPLICATION_CREDENTIALS set "BAZEL_EXTRA_ARGS=--remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }} --google_default_credentials --remote_upload_local_results=true --experimental_remote_cache_compression"
           set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC
           bazelisk build //:xcore-opt --verbose_failures --local_ram_resources=5120 --action_env PYTHON_BIN_PATH="C:/hostedtoolcache/windows/Python/3.8.10/x64/python.exe" %BAZEL_EXTRA_ARGS% --//:disable_version_check

diff --git a/.github/workflows/release-beta.yml b/.github/workflows/release-beta.yml
@@ -89,7 +89,7 @@ jobs:
           cd $GITHUB_WORKSPACE
           make build
 
-          cd $GITHUB_WORKSPACE/experimental/xformer
+          cd $GITHUB_WORKSPACE/xformer
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
             echo "using Google Cloud Storage cache"
             echo -e 'build --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }}' >> .bazelrc.user
@@ -152,7 +152,7 @@ jobs:
           cd $GITHUB_WORKSPACE
           make build
 
-          cd $GITHUB_WORKSPACE/experimental/xformer
+          cd $GITHUB_WORKSPACE/xformer
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
             echo "using Google Cloud Storage cache"
             echo -e 'build --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-arm64-python${{ matrix.python-version }}' >> .bazelrc.user
@@ -206,7 +206,7 @@ jobs:
         # auditwheel requires patchelf and repairs the created wheel file to 
         # be compatible for manylinux.
         run: |
-          pushd $GITHUB_WORKSPACE/experimental/xformer
+          pushd $GITHUB_WORKSPACE/xformer
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
             echo "using Google Cloud Storage cache"
             echo -e 'build --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }}' >> .bazelrc.user
@@ -293,7 +293,7 @@ jobs:
           cmake ..
           cmake --build . -t install --parallel --config Release
 
-          cd %GITHUB_WORKSPACE%/experimental/xformer
+          cd %GITHUB_WORKSPACE%/xformer
           set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC
           bazelisk build //:xcore-opt --local_ram_resources=5120 --action_env PYTHON_BIN_PATH="C:/hostedtoolcache/windows/Python/3.8.10/x64/python.exe" --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }} --google_default_credentials --//:disable_version_check
       - name: Build windows wheels

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -52,7 +52,7 @@ jobs:
           cd $GITHUB_WORKSPACE
           make build
                     
-          cd $GITHUB_WORKSPACE/experimental/xformer
+          cd $GITHUB_WORKSPACE/xformer
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
             echo "using Google Cloud Storage cache"
             echo -e 'build --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }}' >> .bazelrc.user
@@ -105,7 +105,7 @@ jobs:
           cd $GITHUB_WORKSPACE
           make build
                     
-          cd $GITHUB_WORKSPACE/experimental/xformer
+          cd $GITHUB_WORKSPACE/xformer
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
             echo "using Google Cloud Storage cache"
             echo -e 'build --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-arm64-python${{ matrix.python-version }}' >> .bazelrc.user
@@ -151,7 +151,7 @@ jobs:
         # auditwheel requires patchelf and repairs the created wheel file to 
         # be compatible for manylinux.
         run: |
-          pushd $GITHUB_WORKSPACE/experimental/xformer
+          pushd $GITHUB_WORKSPACE/xformer
           if [[ -n $GOOGLE_APPLICATION_CREDENTIALS ]]; then
             echo "using Google Cloud Storage cache"
             echo -e 'build --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }}' >> .bazelrc.user
@@ -227,7 +227,7 @@ jobs:
           cmake ..
           cmake --build . -t install --parallel --config Release
                               
-          cd %GITHUB_WORKSPACE%/experimental/xformer
+          cd %GITHUB_WORKSPACE%/xformer
           set BAZEL_VC=C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC
           bazelisk build //:xcore-opt --local_ram_resources=5120 --action_env PYTHON_BIN_PATH="C:/hostedtoolcache/windows/Python/3.8.10/x64/python.exe" --remote_cache=${{ env.BAZEL_CACHE_URL }}/${{ runner.os }}-${{ runner.arch }}-python${{ matrix.python-version }} --google_default_credentials --//:disable_version_check
       - name: Build windows wheels

diff --git a/.gitignore b/.gitignore
@@ -10,7 +10,7 @@
 **/*.eggs/*
 Pipfile.lock
 ai_tools_venv/*
-experimental/xformer/.venv
+xformer/.venv
 
 # Build cruft
 **/.lock*
@@ -57,7 +57,7 @@ python/xmos_ai_tools/xinterpreters/device/include
 *.h5
 *.html
 *training_history.png
-experimental/xformer/model_maker/*.tflite
+xformer/model_maker/*.tflite
 
 # Others
 *.npz

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -54,7 +54,7 @@ pipeline {
                             // build dll_interpreter for python interface
                             sh "make build"
                             // build xformer
-                            dir("experimental/xformer") {
+                            dir("xformer") {
                                 sh "wget https://github.com/bazelbuild/bazelisk/releases/download/v1.16.0/bazelisk-linux-amd64"
                                 sh "chmod +x bazelisk-linux-amd64"
                                 sh "./bazelisk-linux-amd64 build --remote_cache=${env.BAZEL_CACHE_URL} //:xcore-opt --verbose_failures --//:disable_version_check"
@@ -80,7 +80,7 @@ pipeline {
                         stage("Host Test") {
                             steps {
                                 withVenv {
-                                    dir("experimental/xformer") {
+                                    dir("xformer") {
                                         // xformer2 unit tests
                                         sh "./bazelisk-linux-amd64 test --remote_cache=${env.BAZEL_CACHE_URL} //Test:all --verbose_failures --test_output=errors --//:disable_version_check"
                                     }

diff --git a/Makefile b/Makefile
@@ -35,7 +35,7 @@ xformer2_integration_test:
 #**************************
 .PHONY: version_check
 version_check:
-		cd ./experimental/xformer && ./version_check.sh
+		cd ./xformer && ./version_check.sh
 
 .PHONY: build
 build: version_check xcore_interpreters_build
@@ -73,21 +73,21 @@ help:
 
 .PHONY: init_linux
 init_linux:
-	export BAZEL_VERSION=`cat experimental/xformer/.bazelversion` ;\
+	export BAZEL_VERSION=`cat xformer/.bazelversion` ;\
 	curl -fLO "https://github.com/bazelbuild/bazel/releases/download/$${BAZEL_VERSION}/bazel-$${BAZEL_VERSION}-installer-linux-x86_64.sh" && \
 	chmod +x bazel-$${BAZEL_VERSION}-installer-linux-x86_64.sh && \
 	./bazel-$${BAZEL_VERSION}-installer-linux-x86_64.sh --prefix=$$PWD/bazel
 
 .PHONY: init_darwin
 init_darwin:
-	export BAZEL_VERSION=`cat experimental/xformer/.bazelversion` ;\
+	export BAZEL_VERSION=`cat xformer/.bazelversion` ;\
 	curl -fLO "https://github.com/bazelbuild/bazel/releases/download/$${BAZEL_VERSION}/bazel-$${BAZEL_VERSION}-installer-darwin-x86_64.sh" && \
 	chmod +x bazel-$${BAZEL_VERSION}-installer-darwin-x86_64.sh && \
 	./bazel-$${BAZEL_VERSION}-installer-darwin-x86_64.sh --prefix=$$PWD/bazel
 
 .PHONY: init_windows
 init_windows:
-	export BAZEL_VERSION=`cat experimental/xformer/.bazelversion` ;\
+	export BAZEL_VERSION=`cat xformer/.bazelversion` ;\
 	curl -fLO 'https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-windows-x86_64.exe'
 	mv bazel-${BAZEL_VERSION}-windows-x86_64.exe bazel.exe
 
@@ -102,35 +102,35 @@ build_release_linux:
 	    python3 -m venv .venv && \
 	    . .venv/bin/activate && \
 	    pip install -r requirements.txt && \
-	    cd experimental/xformer && ../../bazel/bin/bazel build --remote_cache=http://srv-bri-bld-cache:8080 //:xcore-opt --verbose_failures)
+	    cd xformer && ../bazel/bin/bazel build --remote_cache=http://srv-bri-bld-cache:8080 //:xcore-opt --verbose_failures)
 	rm -rf ../Installs/Linux/External/xformer
 	mkdir -p ../Installs/Linux/External/xformer
-	cp experimental/xformer/bazel-bin/xcore-opt ../Installs/Linux/External/xformer
+	cp xformer/bazel-bin/xcore-opt ../Installs/Linux/External/xformer
 
 .PHONY: build_release_darwin
 build_release_darwin:
 	( python3 -m venv .venv && \
 	  . .venv/bin/activate && \
 	  pip3 install --upgrade pip && \
 	  pip3 install -r requirements.txt && \
-	  cd experimental/xformer && ../../bazel/bin/bazel build --remote_cache=http://srv-bri-bld-cache:8080 --config=darwin_config //:xcore-opt --verbose_failures)
+	  cd xformer && ../bazel/bin/bazel build --remote_cache=http://srv-bri-bld-cache:8080 --config=darwin_config //:xcore-opt --verbose_failures)
 	rm -rf ../Installs/Mac/External/xformer
 	mkdir -p ../Installs/Mac/External/xformer
-	cp experimental/xformer/bazel-bin/xcore-opt ../Installs/Mac/External/xformer
+	cp xformer/bazel-bin/xcore-opt ../Installs/Mac/External/xformer
 
 .PHONY: build_release_windows
 build_release_windows:
 	python3 -m venv .venv
 	(. .venv/bin/activate && pip install -r requirements.txt)
-	(. .venv/bin/activate && cd experimental/xformer && ../../bazel build --remote_cache=http://srv-bri-bld-cache:8080 --config=windows_config //:xcore-opt --verbose_failures)
+	(. .venv/bin/activate && cd xformer && ../bazel build --remote_cache=http://srv-bri-bld-cache:8080 --config=windows_config //:xcore-opt --verbose_failures)
 	mkdir -p ../Installs/Linux/External/xformer
-	cp experimental/xformer/bazel-bin/xcore-opt ../Installs/Windows/External/xformer
+	cp xformer/bazel-bin/xcore-opt ../Installs/Windows/External/xformer
 
 TEST_SCRIPT= \
 (cd xmos_ai_tools/src/xinterpreters/host/ && make build)&& \
 (cd xmos_ai_tools && python3 setup.py bdist_wheel &&\
 pip install ./xmos_ai_tools/dist/*"&& \
-(cd experimental/xformer && ../../bazel/bin/bazel test --remote_cache=http://srv-bri-bld-cache:8080 //Test:all --verbose_failures)&& \
+(cd xformer && ../bazel/bin/bazel test --remote_cache=http://srv-bri-bld-cache:8080 //Test:all --verbose_failures)&& \
 (pytest integration_tests/runner.py --models_path integration_tests/models/non-bnns -n $(NUM_PROCS) --junitxml=integration_non_bnns_junit.xml)&& \
 (pytest integration_tests/runner.py --models_path integration_tests/models/bnns --bnn -n $(NUM_PROCS) --junitxml=integration_bnns_junit.xml)
 

diff --git a/archived/README.md b/archived/README.md
diff --git a/archived/test/README.md b/archived/test/README.md