Bringup automated tests (#6)

* Initial Jenkinsfile

* Build examples and load xtagctl for tests

* Examples produce different bin names and can be run from any dir

* add xtagctl reset

* Test has size passed from Jenkinsfile

* Add analysis script on fail

* Bump version

* API update with limitation

* Update changelog

CHANGELOG.rst

@@ -1,6 +1,12 @@
 xscope_fileo
 ============
 
+0.3.0
+-----
+  * ADDED: Jenkins driven file transfer test
+  * ADDED: Note in API about maximum fread of 64kB on Linux hosts
+  * FIXED: Duplicate printing of [DEVICE] on long lines
+
 0.2.0
 -----
 

Jenkinsfile

@@ -0,0 +1,124 @@
+@Library('[email protected]') _
+
+getApproval()
+
+pipeline {
+  agent {
+    label 'xcore.ai-explorer'
+  }
+  parameters {
+    string(
+      name: 'TOOLS_VERSION',
+      defaultValue: '15.0.5',
+      description: 'The tools version to build with (check /projects/tools/ReleasesTools/)'
+    )
+  }
+  environment {
+    // '/XMOS/tools' from get_tools.py and rest from tools installers
+    TOOLS_PATH = "/XMOS/tools/${params.TOOLS_VERSION}/XMOS/XTC/${params.TOOLS_VERSION}"
+  }
+  options {
+    skipDefaultCheckout()
+  }
+  stages {
+    stage('Checkout') {
+      steps {
+        checkout scm
+        sh "git clone [email protected]:xmos-int/xtagctl.git"
+      }
+    }
+    stage('Install Dependencies') {
+      steps {
+        sh '/XMOS/get_tools.py ' + params.TOOLS_VERSION
+        toolsEnv(TOOLS_PATH) {
+          installDependencies()
+        }
+      }
+    }
+    stage('Static analysis') {
+      steps {
+        withVenv() {
+          sh "flake8 --exit-zero --output-file=flake8.xml xscope_fileio"
+          recordIssues enabledForFailure: true, tool: flake8(pattern: 'flake8.xml')
+        }
+      }
+    }
+    stage('Build') {
+      steps {
+        toolsEnv(TOOLS_PATH) {  // load xmos tools
+          sh 'tree'
+          sh 'cd examples/throughput_c && make'
+          sh 'cd examples/fileio_features_xc && xmake'
+        }
+      }
+    }
+    stage('Cleanup xtagctl'){
+      steps {
+        withVenv() {
+          toolsEnv(TOOLS_PATH) {
+            sh 'xtagctl reset_all XCORE-AI-EXPLORER'
+            sh 'rm -f ~/.xtag/status.lock ~/.xtag/acquired'
+          }
+        }
+      }
+    }
+    stage('Tests'){
+      failFast false
+      parallel {
+        stage('Hardware tests') {
+          stages{
+            stage('Transfer test single large'){
+              steps {
+                withVenv() {
+                  toolsEnv(TOOLS_PATH) {
+                    sh 'python tests/test_throughput.py 64' //Pass size in MB
+                  }
+                }
+              }
+            }
+          }
+        }
+        stage('Hardware tests #2 (in parallel)') {
+          stages{
+            stage('Transfer test multiple small'){
+              steps {
+                withVenv() {
+                  toolsEnv(TOOLS_PATH) {
+                    sh 'python tests/test_throughput.py 5' //Pass size in MB
+                    sh 'python tests/test_throughput.py 5' //Pass size in MB
+                    sh 'python tests/test_throughput.py 5' //Pass size in MB
+                    sh 'python tests/test_throughput.py 5' //Pass size in MB
+                  }
+                }
+              }
+            }
+          }
+        }
+        stage('xsim tests'){
+          stages{
+            stage('feature test'){
+              steps {
+                withVenv() {
+                  toolsEnv(TOOLS_PATH) {
+                    sh 'python tests/test_features.py'
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  post {
+    success {
+      println "Do some sort of promotion: viewfiles/submodules/branches/tags"
+    }
+    always {
+      archiveArtifacts artifacts: "**/*.bin", fingerprint: true, allowEmptyArchive: true
+    }
+    cleanup {
+      cleanWs()
+    }
+  }
+}

README.md

@@ -49,6 +49,7 @@ void xscope_io_init(chanend_t xscope_end);
 
 xscope_file_t xscope_open_file(char* filename, char* attributes);
 
+//NOTE MAXIMUM n_bytes_to_read of 64kB on Linux http://bugzilla/show_bug.cgi?id=18528
 size_t xscope_fread(xscope_file_t *xscope_io_handle, uint8_t *buffer, size_t n_bytes_to_read);
 
 void xscope_fwrite(xscope_file_t *xscope_io_handle, uint8_t *buffer, size_t n_bytes_to_write);

examples/fileio_features_xc/run_example.py

@@ -1,20 +1,21 @@
 #We assume that the Xscope FileIO Python library has been installed via pip beforehand and is available to import. Please see readme for instuctions.
 import subprocess
 import xscope_fileio
+import os
 
 
-firmware_xe = "bin/fileio_features_xc.xe"
+firmware_xe = os.path.dirname(os.path.realpath(__file__)) + "/bin/fileio_features_xc.xe"
 adapter_id = "L4Ss6YfM"
 ref_text = b"Evolution is change in the heritable characteristics of biological populations over successive generations." + b"\x00";
-with open("ref.bin", "wb") as ref_file:
+with open("features_ref.bin", "wb") as ref_file:
     ref_file.write(ref_text)
 
 xscope_fileio.run_on_target(adapter_id, firmware_xe, use_xsim=True)
 
-with open("dut.bin", "rb") as dut_file:
+with open("features_dut.bin", "rb") as dut_file:
     dut_text = dut_file.read()
 
-with open("dut_mod.bin", "rb") as dut_mod_file:
+with open("features_dut_mod.bin", "rb") as dut_mod_file:
     dut_mod_text = dut_mod_file.read()
 
 ref_mod_text = ref_text[0:10] + b"IS" + ref_text[12:]

examples/fileio_features_xc/test.xc

@@ -8,7 +8,7 @@ extern "C"{
 #include <assert.h>
 
 void test(void){
-    const char ref_file_name[] = "ref.bin";
+    const char ref_file_name[] = "features_ref.bin";
     const char ref_array[] = "Evolution is change in the heritable characteristics of biological populations over successive generations.";
 
     xscope_file_t read_xscope_file = xscope_open_file(ref_file_name, "rb");
@@ -47,12 +47,12 @@ void test(void){
     printf("Full sentence (%u): %s\n", num_bytes, buffer);
 
     //Copy it out to dut for comparing
-    const char dut_file_name[] = "dut.bin";
+    const char dut_file_name[] = "features_dut.bin";
     xscope_file_t write_xscope_file = xscope_open_file(dut_file_name, "wb");
     xscope_fwrite(&write_xscope_file, buffer, sizeof(ref_array));
 
     //Test fseek on write file
-    const char dut_mod_file_name[] = "dut_mod.bin";
+    const char dut_mod_file_name[] = "features_dut_mod.bin";
     xscope_file_t write_xscope_fil_mod = xscope_open_file(dut_mod_file_name, "wb");
     xscope_fwrite(&write_xscope_fil_mod, buffer, sizeof(ref_array));
     xscope_fseek(&write_xscope_fil_mod, 10, SEEK_SET);  

examples/throughput_c/run_example.py

@@ -1,16 +1,18 @@
 #We assume that the Xscope FileIO Python library has been installed via pip beforehand and is available to import. Please see readme for instuctions.
 import numpy as np
 import xscope_fileio
+import os
 
-firmware_xe = "fileio_test.xe"
-adapter_id = "L4Ss6YfM"
+
+firmware_xe = os.path.dirname(os.path.realpath(__file__)) + "/fileio_test.xe"
+adapter_id = "L4Ss6YfM" #enter your xtag id here. Use xrun -l to find out what it is
 
 ref = np.random.randint(256, size=(30 * 1024 * 1024)).astype(np.uint8)
-ref.tofile("ref.bin")
+ref.tofile("throughput_ref.bin")
 
 xscope_fileio.run_on_target(adapter_id, firmware_xe, use_xsim=False)
 
-dut = np.fromfile("dut.bin", dtype=np.uint8)
+dut = np.fromfile("throughput_dut.bin", dtype=np.uint8)
 
 assert np.array_equal(ref, dut)
 

examples/throughput_c/test.c

@@ -8,8 +8,8 @@
 
 #include <assert.h>
 
-#define IN_FILE_NAME    "ref.bin"
-#define OUT_FILE_NAME   "dut.bin"
+#define IN_FILE_NAME    "throughput_ref.bin"
+#define OUT_FILE_NAME   "throughput_dut.bin"
 
 float ticks_to_KBPS(unsigned ticks, unsigned num_bytes){
     const float ticks_per_second = 100000000;
@@ -23,7 +23,7 @@ void do_test(void){
     xscope_file_t read_xscope_file = xscope_open_file(IN_FILE_NAME, "rb");
     xscope_file_t write_xscope_file = xscope_open_file(OUT_FILE_NAME, "wb");
 
-    uint8_t buffer[256*1024] = {0};
+    uint8_t buffer[64*1024] = {0};
 
     xscope_fseek(&read_xscope_file, 0, SEEK_END);
     unsigned fsize = xscope_ftell(&read_xscope_file);

requirements.txt

@@ -0,0 +1,13 @@
+# python_version 3.7
+
+# dependencies
+numpy
+flake8==3.8.3
+pylint==2.5.3
+pytest==6.0.0
+pytest-xdist==1.34.0
+
+# self-install
+-e ./
+-e xtagctl/
+

tests/compare_bins.py

@@ -0,0 +1,38 @@
+import numpy as np
+import sys
+
+
+
+def analyse_error_rate(ref, dut):
+    assert ref.shape == dut.shape, f"Unequal sizes: {ref.shape} {dut.shape}"
+    size = ref.shape[0]
+    num_diff = np.count_nonzero(ref != dut)
+    if num_diff == 0:
+        print("Files identical")
+        return
+    bit_error_ppm = size / 1000000.0 * num_diff
+    print(f"Total bytes: {size} ({size // (1024*1024)}MB), differing bytes: {num_diff}, Error PPM: {bit_error_ppm:.2f}")
+
+    clusters = 0
+    total_cluster_size = 0
+    diff_counter = 0
+    cluster_size_threshold = 16
+    for idx in range(size):
+        if ref[idx] != dut[idx]:
+            if diff_counter == 0:
+                clusters += 1
+                diff_counter = cluster_size_threshold
+            total_cluster_size += 1
+        if diff_counter > 0:
+            diff_counter -= 1
+
+    average_cluster_size = total_cluster_size / clusters
+    print(f"clusters: {clusters}, minimum gap {cluster_size_threshold}, ave_cluster_size: {average_cluster_size}")
+
+if __name__ == "__main__":
+    if len(sys.argv) != 3:
+        assert False, f"usage: {sys.argv[0]} <ref.bin> <dut.bin>"
+    ref = np.fromfile(sys.argv[1], dtype=np.uint8)
+    dut = np.fromfile(sys.argv[2], dtype=np.uint8)
+
+    analyse_error_rate(ref, dut)

tests/test_features.py

@@ -0,0 +1,45 @@
+#We assume that the Xscope FileIO Python library has been installed via pip beforehand and is available to import. Please see readme for instuctions.
+import numpy as np
+import xscope_fileio
+import os, sys
+import xtagctl
+import contextlib
+import random, string
+
+
+@contextlib.contextmanager
+def cd(path):
+    CWD = os.getcwd()
+    os.mkdir(path)
+    os.chdir(path)
+    yield
+    os.chdir(CWD)
+
+
+def run_features():
+    tmpdirname = 'tmp_features_' + ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))
+    test_dir = os.path.dirname(os.path.realpath(__file__)) + "/" + tmpdirname
+    with cd(test_dir):
+        ref_text = b"Evolution is change in the heritable characteristics of biological populations over successive generations." + b"\x00";
+        with open("features_ref.bin", "wb") as ref_file:
+            ref_file.write(ref_text)
+
+        firmware_xe = test_dir + "/../../examples/fileio_features_xc/bin/fileio_features_xc.xe"
+        xscope_fileio.run_on_target(None, firmware_xe, use_xsim=True)
+
+
+        with open("features_dut.bin", "rb") as dut_file:
+            dut_text = dut_file.read()
+
+        with open("features_dut_mod.bin", "rb") as dut_mod_file:
+            dut_mod_text = dut_mod_file.read()
+
+        ref_mod_text = ref_text[0:10] + b"IS" + ref_text[12:]
+
+        assert dut_text == ref_text
+        print(dut_mod_text, ref_mod_text)
+        assert dut_mod_text == ref_mod_text
+    print("PASS")
+
+if __name__ == "__main__":
+    run_features()

tests/test_throughput.py

@@ -0,0 +1,45 @@
+#We assume that the Xscope FileIO Python library has been installed via pip beforehand and is available to import. Please see readme for instuctions.
+import numpy as np
+import xscope_fileio
+import os, sys
+import xtagctl
+import contextlib
+import random, string
+from compare_bins import analyse_error_rate
+
+
+@contextlib.contextmanager
+def cd(path):
+    CWD = os.getcwd()
+    os.mkdir(path)
+    os.chdir(path)
+    yield
+    os.chdir(CWD)
+
+
+def run_throughput(size_mb):
+    tmpdirname = 'tmp_throughput_' + ''.join(random.choices(string.ascii_uppercase + string.digits, k=8))
+    test_dir = os.path.dirname(os.path.realpath(__file__)) + "/" + tmpdirname
+    with cd(test_dir):
+        ref = np.random.randint(256, size=(size_mb * 1024 * 1024)).astype(np.uint8)
+        ref.tofile("throughput_ref.bin")
+
+        with xtagctl.acquire("XCORE-AI-EXPLORER", timeout=10) as adapter_id:
+            firmware_xe = test_dir + "/../../examples/throughput_c/fileio_test.xe"
+            xscope_fileio.run_on_target(adapter_id, firmware_xe, use_xsim=False)
+
+        dut = np.fromfile("throughput_dut.bin", dtype=np.uint8)
+
+        equal = np.array_equal(ref, dut)
+        if not equal:
+            analyse_error_rate(ref, dut)
+            assert 0
+
+    print("PASS")
+
+if __name__ == "__main__":
+    if len(sys.argv) > 1:
+        size_mb = int(sys.argv[1])
+    else:
+        size_mb =  30
+    run_throughput(size_mb)

xscope_fileio/api/xscope_io_device.h

@@ -49,7 +49,7 @@ void xscope_io_init(chanend_t xscope_end);
 xscope_file_t xscope_open_file(const char* filename, char* attributes);
 
 /******************************************************************************
- * xscope_fread
+ * xscope_fread   NOTE MAXIMUM n_bytes_to_read of 64kB on Linux, bugzillaid=18528
  *
  * Reads a number of bytes into the buffer provided by the application.
  * It sends a command to the host app which responds with an upload of the