From e6b4c07f72bfcc2d7e0aad21ab98581b31e8a809 Mon Sep 17 00:00:00 2001 From: Arjun Suresh Date: Wed, 1 May 2024 14:50:42 +0530 Subject: [PATCH] Removed duplicated tests directory --- tests/script/check.py | 14 ---------- tests/script/process_dockerfile.py | 30 --------------------- tests/script/process_readme.py | 26 ------------------ tests/script/test_deps.py | 23 ---------------- tests/script/test_docker.py | 21 --------------- tests/script/test_features.py | 21 --------------- tests/script/test_install.py | 10 ------- tests/test_cm.py | 14 ---------- tests/test_search_speed.py | 25 ----------------- tests/tutorials/test_tutorial_retinanet.py | 28 ------------------- tests/tutorials/test_tutorial_tvm.py | 24 ----------------- tests/tutorials/test_tutorial_tvm_pip_ge.py | 21 --------------- tests/tutorials/test_tutorial_tvm_pip_vm.py | 23 ---------------- 13 files changed, 280 deletions(-) delete mode 100644 tests/script/check.py delete mode 100644 tests/script/process_dockerfile.py delete mode 100644 tests/script/process_readme.py delete mode 100644 tests/script/test_deps.py delete mode 100644 tests/script/test_docker.py delete mode 100644 tests/script/test_features.py delete mode 100644 tests/script/test_install.py delete mode 100644 tests/test_cm.py delete mode 100644 tests/test_search_speed.py delete mode 100644 tests/tutorials/test_tutorial_retinanet.py delete mode 100644 tests/tutorials/test_tutorial_tvm.py delete mode 100644 tests/tutorials/test_tutorial_tvm_pip_ge.py delete mode 100644 tests/tutorials/test_tutorial_tvm_pip_vm.py diff --git a/tests/script/check.py b/tests/script/check.py deleted file mode 100644 index 7394406d8..000000000 --- a/tests/script/check.py +++ /dev/null @@ -1,14 +0,0 @@ -def check_return(r): - if 'return' not in r: - raise Exception('CM access function should always return key \'return\'!') - if 'error' in r: - raise Exception(r['error']) - -def check_list(r, string, found=True): - check_return(r) - if 'list' not in r: - raise Exception('CM search should return a list!') - if len(r['list']) < 1 and found: - raise Exception('CM search returned an empty list for ' + string) - if len(r['list']) > 0 and not found: - raise Exception('CM search returned at lease one entry for ' + string) diff --git a/tests/script/process_dockerfile.py b/tests/script/process_dockerfile.py deleted file mode 100644 index 9a58fce38..000000000 --- a/tests/script/process_dockerfile.py +++ /dev/null @@ -1,30 +0,0 @@ -import sys -import os -import cmind as cm -import check as checks -import json -import yaml - -files=sys.argv[1:] - -for file in files: - if not os.path.isfile(file): - continue - if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"): - continue - if not file.startswith(os.path.join("cm-mlops", "script")): - continue - - script_path = os.path.dirname(file) - - f = open(file) - - if file.endswith(".json"): - data = json.load(f) - elif file.endswith(".yaml"): - data = yaml.safe_load(f) - - uid = data['uid'] - - r = cm.access({'action':'dockerfile', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) - checks.check_return(r) diff --git a/tests/script/process_readme.py b/tests/script/process_readme.py deleted file mode 100644 index 492813a05..000000000 --- a/tests/script/process_readme.py +++ /dev/null @@ -1,26 +0,0 @@ -import sys -import os -import cmind as cm -import check as checks -import json -import yaml - -files=sys.argv[1:] - -for file in files: - if not os.path.isfile(file): - continue - if not file.endswith("_cm.json") and not file.endswith("_cm.yaml"): - continue - if not file.startswith(os.path.join("cm-mlops", "script")): - continue - script_path = os.path.dirname(file) - f = open(file) - if file.endswith(".json"): - data = json.load(f) - elif file.endswith(".yaml"): - data = yaml.safe_load(f) - uid = data['uid'] - - r = cm.access({'action':'doc', 'automation':'script', 'artifact': uid, 'quiet': 'yes'}) - checks.check_return(r) diff --git a/tests/script/test_deps.py b/tests/script/test_deps.py deleted file mode 100644 index aaf19bc81..000000000 --- a/tests/script/test_deps.py +++ /dev/null @@ -1,23 +0,0 @@ -# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps - -import cmind as cm -import check as checks - -# MLPerf v3.0 inference is now very outdated and we are testing inference in separate tests - -#r = cm.access({'action':'run', 'automation':'script', 'tags': 'generate-run-cmds,mlperf', 'adr': -# {'loadgen': {'version': 'r3.0'}, 'compiler': {'tags': "gcc"}}, 'env': {'CM_MODEL': 'resnet50', -# 'CM_DEVICE': 'cpu', 'CM_BACKEND': 'onnxruntime'}, 'quiet': 'yes'}) -#checks.check_return(r) -# -#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'loadgen,version-r3.0,deps-python-non-virtual'}) -#checks.check_list(r, "loadgen,version-r3.0,deps-python-non-virtual") -# -#r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'inference,src,version-r3.0'}) -#checks.check_list(r, "inference,src,version-r3.0") -# -#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_onnxruntime,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) -#checks.check_return(r) -# -#r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_python,_resnet50,_tf,_cpu,_r3.0_default', 'adr': {'mlperf-implementation': { 'version': 'master'}}, 'quiet': 'yes'}) -#checks.check_return(r) diff --git a/tests/script/test_docker.py b/tests/script/test_docker.py deleted file mode 100644 index 5a02c932e..000000000 --- a/tests/script/test_docker.py +++ /dev/null @@ -1,21 +0,0 @@ -# This test covers version, variation, compilation from src, add_deps_recursive, post_deps - -import cmind as cm -import check as checks - -r = cm.access({'action':'run', - 'automation':'script', - 'tags': 'run,docker,container', - 'add_deps_recursive': { - 'compiler': {'tags': "gcc"} - }, - 'image_name':'cm-script-app-image-classification-onnx-py', - 'env': { - 'CM_DOCKER_RUN_SCRIPT_TAGS': 'app,image-classification,onnx,python', - 'CM_MLOPS_REPO': 'ctuning@mlcommons-ck', - 'CM_DOCKER_IMAGE_BASE': 'ubuntu:22.04' - }, - 'quiet': 'yes' - }) - -checks.check_return(r) diff --git a/tests/script/test_features.py b/tests/script/test_features.py deleted file mode 100644 index d116cbd5b..000000000 --- a/tests/script/test_features.py +++ /dev/null @@ -1,21 +0,0 @@ -# This test covers -# 1. python-virtual-env and update_deps inside customize.py -# 2. cache search using "-" prefix - -import cmind as cm -import check as checks - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'name':'test', 'quiet': 'yes'}) -checks.check_return(r) - -r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,python,virtual,name-test'}) -checks.check_list(r, "get,python-venv") - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'get,dataset,preprocessed,imagenet,_NHWC', 'quiet': 'yes'}) -checks.check_return(r) - -r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NCHW'}) -checks.check_list(r, "_NHWC") - -r = cm.access({'action':'search', 'automation': 'cache', 'tags': 'get,dataset,preprocessed,imagenet,-_NHWC'}) -checks.check_list(r, "_NHWC", False) diff --git a/tests/script/test_install.py b/tests/script/test_install.py deleted file mode 100644 index 66fa164d5..000000000 --- a/tests/script/test_install.py +++ /dev/null @@ -1,10 +0,0 @@ -# This test covers script installation, version, shared library install - -import cmind as cm -import check as checks - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'python,src,install,_shared', 'version': '3.9.10', 'quiet': 'true'}) -checks.check_return(r) - -r = cm.access({'action':'search', 'automation':'cache', 'tags': 'python,src,install,_shared,version-3.9.10'}) -checks.check_list(r, "python,src,install,_shared,version-3.9.10") diff --git a/tests/test_cm.py b/tests/test_cm.py deleted file mode 100644 index 41fb402c2..000000000 --- a/tests/test_cm.py +++ /dev/null @@ -1,14 +0,0 @@ -try: - import cmind as cm - - r = cm.access(['test', 'script']) - if 'return' not in r: - raise Exception('CM access function should always return key \'return\'!') - exit(0) - -except ImportError as e: - from sys import stderr - from subprocess import call - print('WARNING: CM module for python is not installed & jupyter notebooks will not be supported', file=stderr) - retcode = call(['cm', 'test', 'script']) - exit(retcode) diff --git a/tests/test_search_speed.py b/tests/test_search_speed.py deleted file mode 100644 index 3086a8340..000000000 --- a/tests/test_search_speed.py +++ /dev/null @@ -1,25 +0,0 @@ -import cmind as cm -import time - -times = [] - -steps = 10 - -print ('Running search with tags {} times ...'.format(steps)) - -for step in range(steps): - - start = time.time() - r = cm.access({'action':'search', - 'automation':'script', - 'tags':'detect,os'}) - timer = time.time() - start - - if r['return']>0: cm.error(r) - - times.append(timer) - -step = 0 -for t in times: - step += 1 - print ("{}) {:0.3f} sec.".format(step, t)) diff --git a/tests/tutorials/test_tutorial_retinanet.py b/tests/tutorials/test_tutorial_retinanet.py deleted file mode 100644 index 404b69fe9..000000000 --- a/tests/tutorials/test_tutorial_retinanet.py +++ /dev/null @@ -1,28 +0,0 @@ -# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps - -import cmind as cm -from pathlib import Path -import sys -import os - -sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) -import check as checks - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \ - {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \ - 'mode': 'accuracy', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) -checks.check_return(r) - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'app,mlperf,inference,generic,_cpp,_retinanet,_onnxruntime,_cpu', 'adr': \ - {'python': {'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'scenario': 'Offline', \ - 'mode': 'performance', 'test_query_count': '10', 'rerun': 'true', 'quiet': 'yes'}) -checks.check_return(r) - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'install,python-venv', 'version': '3.10.8', 'name': 'mlperf' }) -checks.check_return(r) - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ - {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'compiler': {'tags': "gcc"}, 'openimages-preprocessed': {'tags': '_50'}}, 'submitter': 'Community', \ - 'implementation': 'cpp', 'hw_name': 'default', 'model': 'retinanet', 'backend': 'onnxruntime', 'device': 'cpu', 'scenario': 'Offline', \ - 'test_query_count': '10', 'clean': 'true', 'quiet': 'yes'}) -checks.check_return(r) diff --git a/tests/tutorials/test_tutorial_tvm.py b/tests/tutorials/test_tutorial_tvm.py deleted file mode 100644 index 930e3622d..000000000 --- a/tests/tutorials/test_tutorial_tvm.py +++ /dev/null @@ -1,24 +0,0 @@ -# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps - -import cmind as cm - -from pathlib import Path -import sys -import os - -sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) -import check as checks - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \ - {'python': {'name': 'mlperf', 'version_min': '3.8'}}, 'submitter': 'Community', \ - 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ - 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) -checks.check_return(r) - - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_dashboard', 'adr': \ - {'python': {'name': 'mlperf', 'version_min': '3.8'}}, 'submitter': 'Community', \ - 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ - 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) -checks.check_return(r) - diff --git a/tests/tutorials/test_tutorial_tvm_pip_ge.py b/tests/tutorials/test_tutorial_tvm_pip_ge.py deleted file mode 100644 index 0c9a4b9c3..000000000 --- a/tests/tutorials/test_tutorial_tvm_pip_ge.py +++ /dev/null @@ -1,21 +0,0 @@ -import cmind as cm - -from pathlib import Path -import sys -import os - -sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) -import check as checks - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \ - {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}, 'tvm-model': {'tags': '_graph_executor'}}, \ - 'submitter': 'Community', 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', \ - 'device': 'cpu', 'scenario': 'Offline', 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) -checks.check_return(r) - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ - {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}, 'tvm-model': {'tags': '_graph_executor'}}, \ - 'submitter': 'Community', 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', \ - 'device': 'cpu', 'scenario': 'Offline', 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) -checks.check_return(r) - diff --git a/tests/tutorials/test_tutorial_tvm_pip_vm.py b/tests/tutorials/test_tutorial_tvm_pip_vm.py deleted file mode 100644 index 81069194d..000000000 --- a/tests/tutorials/test_tutorial_tvm_pip_vm.py +++ /dev/null @@ -1,23 +0,0 @@ -# This test covers version, variation, compilation from src, add_deps, add_deps_recursive, deps, post_deps - -import cmind as cm - -from pathlib import Path -import sys -import os - -sys.path.insert(1, os.path.join(Path(__file__).parent.parent.resolve(), "script")) -import check as checks - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds', 'adr': \ - {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}}, 'submitter': 'Community', \ - 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ - 'mode': 'accuracy', 'test_query_count': '5', 'clean': 'true', 'quiet': 'yes'}) -checks.check_return(r) - -r = cm.access({'action':'run', 'automation':'script', 'tags': 'run,mlperf,inference,generate-run-cmds,_submission,_short,_dashboard', 'adr': \ - {'python': {'name': 'mlperf', 'version_min': '3.8'}, 'tvm': {'tags': '_pip-install'}}, 'submitter': 'Community', \ - 'implementation': 'python', 'hw_name': 'default', 'model': 'resnet50', 'backend': 'tvm-onnx', 'device': 'cpu', 'scenario': 'Offline', \ - 'test_query_count': '500', 'clean': 'true', 'quiet': 'yes'}) -checks.check_return(r) -