diff --git a/CHANGES.md b/CHANGES.md index c49755977..d640f53ee 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -1,3 +1,8 @@ +### 20240927 + * added "test dummy" script to test Docker containers + * added more standard Nvidia Docker configuration for PyTorch + * added better support to select Docker configurations via UID + ### 20240916 * fixed "cm add script" diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 80e8c506b..7b62e067d 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -1,5 +1,5 @@ Copyright (c) 2021-2024 MLCommons -The cTuning foundation and OctoML donated this project to MLCommons to benefit everyone. +Grigori Fursin, the cTuning foundation and OctoML donated this project to MLCommons to benefit everyone. Copyright (c) 2014-2021 cTuning foundation diff --git a/README.md b/README.md index 777340d39..06c2cd2bc 100644 --- a/README.md +++ b/README.md @@ -139,6 +139,11 @@ cm run script \ [Apache 2.0](LICENSE.md) +## CM concepts + +* https://doi.org/10.5281/zenodo.8105339 +* https://arxiv.org/abs/2406.16791 + ## Authors [Grigori Fursin](https://cKnowledge.org/gfursin) and [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) diff --git a/automation/script/module.py b/automation/script/module.py index e40cf8b35..9494e7a39 100644 --- a/automation/script/module.py +++ b/automation/script/module.py @@ -4028,14 +4028,58 @@ def docker(self, i): (out) (str): if 'con', output to console - parsed_artifact (list): prepared in CM CLI or CM access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - (repos) (str): list of repositories to search for automations (output_dir) (str): output directory (./ by default) + (docker) (dict): convert keys into docker_{key} strings for CM >= 2.3.8.1 + + + (docker_skip_build) (bool): do not generate Dockerfiles and do not recreate Docker image (must exist) + (docker_noregenerate) (bool): do not generate Dockerfiles + (docker_norecreate) (bool): do not recreate Docker image + + (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one + (docker_cfg_uid) (str): if True, select docker configuration with this UID + + (docker_path) (str): where to create or find Dockerfile + (docker_gh_token) (str): GitHub token for private repositories + (docker_save_script) (str): if !='' name of script to save docker command + (docker_interactive) (bool): if True, run in interactive mode + (docker_it) (bool): the same as `docker_interactive` + (docker_detached) (bool): detach Docker + (docker_dt) (bool) the same as `docker_detached` + + (docker_base_image) (str): force base image + (docker_os) (str): force docker OS (default: ubuntu) + (docker_os_version) (str): force docker OS version (default: 22.04) + (docker_image_tag_extra) (str): add extra tag (default:-latest) + + (docker_cm_repo) (str): force CM automation repository when building Docker (default: cm4mlops) + (docker_cm_repos) + (docker_cm_repo_flags) + + (dockerfile_env) + + (docker_skip_cm_sys_upgrade) (bool): if True, do not install CM sys deps + + (docker_extra_sys_deps) + + (fake_run_deps) + (docker_run_final_cmds) + + (all_gpus) + (num_gpus) + + (docker_device) + + (docker_port_maps) + + (docker_shm_size) + + (docker_extra_run_args) + + Returns: (CM return dict): diff --git a/automation/script/module_misc.py b/automation/script/module_misc.py index b193ff5df..b68d90e27 100644 --- a/automation/script/module_misc.py +++ b/automation/script/module_misc.py @@ -1335,15 +1335,9 @@ def dockerfile(i): Args: (CM input dict): - (out) (str): if 'con', output to console - - parsed_artifact (list): prepared in CM CLI or CM access function - [ (artifact alias, artifact UID) ] or - [ (artifact alias, artifact UID), (artifact repo alias, artifact repo UID) ] - - (repos) (str): list of repositories to search for automations - - (output_dir) (str): output directory (./ by default) + (out) (str): if 'con', output to console + (repos) (str): list of repositories to search for automations + (output_dir) (str): output directory (./ by default) Returns: (CM return dict): @@ -1632,15 +1626,6 @@ def docker(i): (out) (str): if 'con', output to console - (docker_skip_build) (bool): do not generate Dockerfiles and do not recreate Docker image (must exist) - (docker_noregenerate) (bool): do not generate Dockerfiles - (docker_norecreate) (bool): do not recreate Docker image - - (docker_path) (str): where to create or find Dockerfile - (docker_gh_token) (str): GitHub token for private repositories - (docker_save_script) (str): if !='' name of script to save docker command - (docker_interactive) (bool): if True, run in interactive mode - (docker_cfg) (str): if True, show all available basic docker configurations, otherwise pre-select one Returns: (CM return dict): @@ -1653,6 +1638,20 @@ def docker(i): import copy import re + from cmind import __version__ as current_cm_version + + self_module = i['self_module'] + + if type(i.get('docker', None)) == dict: + # Grigori started cleaning and refactoring this code on 20240929 + # + # 1. use --docker dictionary instead of --docker_{keys} + + if utils.compare_versions(current_cm_version, '2.3.8.1') >= 0: + docker_params = utils.convert_dictionary(i['docker'], 'docker') + i.update(docker_params) + del(i['docker']) + quiet = i.get('quiet', False) detached = i.get('docker_detached', '') @@ -1670,13 +1669,12 @@ def docker(i): # Check simplified CMD: cm docker script "python app image-classification onnx" # If artifact has spaces, treat them as tags! - self_module = i['self_module'] self_module.cmind.access({'action':'detect_tags_in_artifact', 'automation':'utils', 'input':i}) # CAREFUL -> artifacts and parsed_artifacts are not supported in input (and should not be?) if 'artifacts' in i: del(i['artifacts']) if 'parsed_artifacts' in i: del(i['parsed_artifacts']) - + # Prepare "clean" input to replicate command r = self_module.cmind.access({'action':'prune_input', 'automation':'utils', 'input':i, 'extra_keys_starts_with':['docker_']}) i_run_cmd_arc = r['new_input'] @@ -1693,13 +1691,19 @@ def docker(i): # Check available configurations docker_cfg = i.get('docker_cfg', '') - if docker_cfg != '': + docker_cfg_uid = i.get('docker_cfg_uid', '') + + if docker_cfg != '' or docker_cfg_uid != '': # Check if docker_cfg is turned on but not selected if type(docker_cfg) == bool or str(docker_cfg).lower() in ['true','yes']: docker_cfg= '' - - r = self_module.cmind.access({'action':'select_cfg', 'automation':'utils,dc2743f8450541e3', - 'tags':'basic,docker,configurations', 'title':'docker', 'alias':docker_cfg}) + + r = self_module.cmind.access({'action':'select_cfg', + 'automation':'utils,dc2743f8450541e3', + 'tags':'basic,docker,configurations', + 'title':'docker', + 'alias':docker_cfg, + 'uid':docker_cfg_uid}) if r['return'] > 0: if r['return'] == 16: return {'return':1, 'error':'Docker configuration {} was not found'.format(docker_cfg)} @@ -1708,10 +1712,9 @@ def docker(i): selection = r['selection'] docker_input_update = selection['meta']['input'] - + i.update(docker_input_update) - ######################################################################################## # Run dockerfile if not noregenerate_docker_file: @@ -1722,7 +1725,7 @@ def docker(i): cur_dir = os.getcwd() console = i.get('out') == 'con' - + # Search for script(s) r = aux_search({'self_module': self_module, 'input': i}) if r['return']>0: return r diff --git a/automation/utils/module_cfg.py b/automation/utils/module_cfg.py index 145c388f2..04ab0a9ad 100644 --- a/automation/utils/module_cfg.py +++ b/automation/utils/module_cfg.py @@ -230,16 +230,18 @@ def select_cfg(i): self_module = i['self_module'] tags = i['tags'] alias = i.get('alias', '') + uid = i.get('uid', '') title = i.get('title', '') # Check if alias is not provided r = self_module.cmind.access({'action':'find', 'automation':'cfg', 'tags':'basic,docker,configurations'}) if r['return'] > 0: return r - + lst = r['list'] selector = [] + # Do coarse-grain search for CM artifacts for l in lst: p = l.path @@ -257,45 +259,53 @@ def select_cfg(i): if not f.startswith('_cm') and (f.endswith('.json') or f.endswith('.yaml')): selector.append({'path':os.path.join(p, f), 'alias':f[:-5]}) - if len(selector) == 0: - return {'return':16, 'error':'configuration was not found'} - select = 0 - if len(selector) > 1: - xtitle = ' ' + title if title!='' else '' - print ('') - print ('Available{} configurations:'.format(xtitle)) - - print ('') + # Load meta for name and UID + selector_with_meta = [] + for s in range(0, len(selector)): + ss = selector[s] - for s in range(0, len(selector)): - ss = selector[s] + path = ss['path'] - path = ss['path'] + full_path_without_ext = path[:-5] - full_path_without_ext = path[:-5] + r = cmind.utils.load_yaml_and_json(full_path_without_ext) + if r['return']>0: + print ('Warning: problem loading configuration file {}'.format(path)) - r = cmind.utils.load_yaml_and_json(full_path_without_ext) - if r['return']>0: - print ('Warning: problem loading configuration file {}'.format(path)) + meta = r['meta'] - meta = r['meta'] + if uid == '' or meta.get('uid', '') == uid: ss['meta'] = meta + selector_with_meta.append(ss) + + # Quit if no configurations found + if len(selector_with_meta) == 0: + return {'return':16, 'error':'configuration was not found'} - selector = sorted(selector, key = lambda x: x['meta'].get('name','')) + select = 0 + if len(selector_with_meta) > 1: + xtitle = ' ' + title if title!='' else '' + print ('') + print ('Available{} configurations:'.format(xtitle)) + + print ('') + + selector_with_meta = sorted(selector_with_meta, key = lambda x: x['meta'].get('name','')) s = 0 - for ss in selector: + for ss in selector_with_meta: alias = ss['alias'] - name = ss['meta'].get('name','') + uid = ss['meta'].get('uid', '') + name = ss['meta'].get('name', '') x = name if x!='': x+=' ' - x += '('+alias+')' - - print ('{}) {}'.format(s, x)) + x += '(' + uid + ')' + + print (f'{s}) {x}'.format(s, x)) s+=1 - + print ('') select = input ('Enter configuration number of press Enter for 0: ') @@ -306,6 +316,6 @@ def select_cfg(i): if select<0 or select>=len(selector): return {'return':1, 'error':'selection is out of range'} - ss = selector[select] + ss = selector_with_meta[select] return {'return':0, 'selection':ss} diff --git a/cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml b/cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml new file mode 100644 index 000000000..716adc20b --- /dev/null +++ b/cfg/benchmark-run-mlperf-inference-v4.1/_cm.yaml @@ -0,0 +1,39 @@ +alias: benchmark-run-mlperf-inference-v4.1 +uid: b7e89771987d4168 + +automation_alias: cfg +automation_uid: 88dce9c160324c5d + +tags: +- benchmark +- run +- mlperf +- inference +- v4.1 + +name: "MLPerf inference - v4.1" + +supported_compute: +- ee8c568e0ac44f2b +- fe379ecd1e054a00 +- d8f06040f7294319 + +bench_uid: 39877bb63fb54725 + +view_dimensions: +- - input.device + - "MLPerf device" +- - input.implementation + - "MLPerf implementation" +- - input.backend + - "MLPerf backend" +- - input.model + - "MLPerf model" +- - input.scenario + - "MLPerf scenario" +- - input.host_os + - "Host OS" +- - output.state.cm-mlperf-inference-results-last.performance + - "Got performance" +- - output.state.cm-mlperf-inference-results-last.accuracy + - "Got accuracy" diff --git a/cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml b/cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml new file mode 100644 index 000000000..d949d5519 --- /dev/null +++ b/cfg/docker-basic-configurations/basic-ubuntu-24.04.yaml @@ -0,0 +1,9 @@ +uid: 12e86eb386314866 + +name: "Basic Ubuntu 24.04" + +input: + docker_base_image: 'ubuntu:24.04' + docker_os: ubuntu + docker_os_version: '24.04' + \ No newline at end of file diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml index 1e71c67ce..16107d8d5 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-20.04-cuda-11.8-cudnn-8.6.0-pytorch-1.13.0.yaml @@ -1,6 +1,8 @@ uid: 854e65fb31584d63 -name: "Nvidia Ubuntu 20.04 CUDA 11.8 cuDNN 8.6.0 PyTorch 1.13.0" +name: "Nvidia Ubuntu 20.04 CUDA 11.8 cuDNN 8.6.0 PyTorch 1.13.0 (pytorch:22.10)" + +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-22-10.html input: docker_base_image: 'nvcr.io/nvidia/pytorch:22.10-py3' diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml index 4e146ca66..66b9efd0d 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.1-cudnn-8.9.1-pytorch-2.0.0.yaml @@ -1,6 +1,8 @@ uid: e0e7167139a74e36 -name: "Nvidia Ubuntu 22.04 CUDA 12.1 cuDNN 8.9.1 PyTorch 2.0.0" +name: "Nvidia Ubuntu 22.04 CUDA 12.1 cuDNN 8.9.1 PyTorch 2.0.0 (pytorch:23.05)" + +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-23-05.html input: docker_base_image: 'nvcr.io/nvidia/pytorch:23.05-py3' diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml index a671d699a..38bcff694 100644 --- a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.4-cudnn-9.0.0-pytorch-2.3.0.yaml @@ -1,6 +1,8 @@ uid: 49fc51f2999b4545 -name: "Nvidia Ubuntu 22.04 CUDA 12.4 cuDNN 9.0.0 PyTorch 2.3.0" +name: "Nvidia Ubuntu 22.04 CUDA 12.4 cuDNN 9.0.0 PyTorch 2.3.0 (pytorch:24.03)" + +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-03.html input: docker_base_image: 'nvcr.io/nvidia/pytorch:24.03-py3' diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml new file mode 100644 index 000000000..b4e45d348 --- /dev/null +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.5-cudnn-9.1.0-pytorch-2.4.0.yaml @@ -0,0 +1,11 @@ +uid: 81879736ae5842f4 + +name: "Nvidia Ubuntu 22.04 CUDA 12.5 cuDNN 9.1.0 PyTorch 2.4.0 (pytorch:24.06)" + +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-06.html + +input: + docker_base_image: 'nvcr.io/nvidia/pytorch:24.06-py3' + docker_os: ubuntu + docker_os_version: '22.04' + \ No newline at end of file diff --git a/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml new file mode 100644 index 000000000..a9e2229ea --- /dev/null +++ b/cfg/docker-basic-configurations/nvidia-ubuntu-22.04-cuda-12.6-cudnn-9.3.0-pytorch-2.5.0.yaml @@ -0,0 +1,11 @@ +uid: 203a68df99d44137 + +name: "Nvidia Ubuntu 22.04 CUDA 12.6 cuDNN 9.3.0 PyTorch 2.5.0 (pytorch:24.08)" + +ref_url: https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-24-08.html + +input: + docker_base_image: 'nvcr.io/nvidia/pytorch:24.08-py3' + docker_os: ubuntu + docker_os_version: '22.04' + \ No newline at end of file diff --git a/debug.py b/debug.py new file mode 100644 index 000000000..9fd54803c --- /dev/null +++ b/debug.py @@ -0,0 +1,9 @@ +# Developer: Grigori Fursin + +import cmind +import sys + +print(sys.executable) + +r = cmind.access('run script "print hello-world python" --debug_uid=f52670e5f3f345a2') +print(r) diff --git a/script/app-image-classification-onnx-py/_cm.yaml b/script/app-image-classification-onnx-py/_cm.yaml index 825be9128..2e2241a07 100644 --- a/script/app-image-classification-onnx-py/_cm.yaml +++ b/script/app-image-classification-onnx-py/_cm.yaml @@ -22,7 +22,7 @@ default_env: deps: - tags: detect,os -- tags: get,sys-utils-cm +#- tags: get,sys-utils-cm - names: - python - python3 diff --git a/script/app-image-classification-torch-py/_cm.json b/script/app-image-classification-torch-py/_cm.json index a6a78a679..4e30d3f5f 100644 --- a/script/app-image-classification-torch-py/_cm.json +++ b/script/app-image-classification-torch-py/_cm.json @@ -11,9 +11,6 @@ { "tags": "detect,os" }, - { - "tags": "get,sys-utils-cm" - }, { "names": [ "python", diff --git a/script/app-image-classification-tvm-onnx-py/_cm.json b/script/app-image-classification-tvm-onnx-py/_cm.json deleted file mode 100644 index 1ae2e5c32..000000000 --- a/script/app-image-classification-tvm-onnx-py/_cm.json +++ /dev/null @@ -1,73 +0,0 @@ -{ - "alias": "app-image-classification-tvm-onnx-py", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "category": "Modular AI/ML application pipeline", - "default_env": { - "CM_BATCH_COUNT": "1", - "CM_BATCH_SIZE": "1" - }, - "deps": [ - { - "tags": "detect,os" - }, - { - "tags": "detect,cpu" - }, - { - "names": [ - "python", - "python3" - ], - "tags": "get,python3" - }, - { - "tags": "get,dataset,image-classification,original" - }, - { - "tags": "get,dataset-aux,image-classification" - }, - { - "tags": "get,raw,ml-model,image-classification,resnet50,_onnx" - }, - { - "tags": "get,generic-python-lib,_onnxruntime" - }, - { - "names": [ "tvm" ], - "tags": "get,tvm" - } - ], - "tags": [ - "app", - "image-classification", - "tvm-onnx", - "python" - ], - "tags_help":"app image-classification python tvm-onnx", - "uid": "63080407db4d4ac4", - "variations": { - "llvm": { - "add_deps_recursive": { - "tvm": { - "tags": "_llvm" - } - } - }, - "cuda": { - "add_deps_recursive": { - "tvm": { - "tags": "_cuda" - } - }, - "env": { - "USE_CUDA": "yes" - }, - "deps": [ - { - "tags": "get,cuda" - } - ] - } - } -} diff --git a/script/app-image-classification-tvm-onnx-py/_cm.yaml b/script/app-image-classification-tvm-onnx-py/_cm.yaml new file mode 100644 index 000000000..2b5cc9cca --- /dev/null +++ b/script/app-image-classification-tvm-onnx-py/_cm.yaml @@ -0,0 +1,48 @@ +alias: app-image-classification-tvm-onnx-py +uid: 63080407db4d4ac4 + +automation_alias: script +automation_uid: 5b4e0237da074764 + +category: Modular AI/ML application pipeline + +default_env: + CM_BATCH_COUNT: '1' + CM_BATCH_SIZE: '1' + +deps: +- tags: detect,os +- tags: detect,cpu +- names: + - python + - python3 + tags: get,python3 +- tags: get,dataset,image-classification,original +- tags: get,dataset-aux,image-classification +- tags: get,raw,ml-model,image-classification,resnet50,_onnx +- tags: get,generic-python-lib,_onnxruntime +- names: + - tvm + tags: get,tvm + +tags: +- app +- image-classification +- tvm-onnx +- python + +tags_help: app image-classification python tvm-onnx + +variations: + cuda: + add_deps_recursive: + tvm: + tags: _cuda + deps: + - tags: get,cuda + env: + USE_CUDA: 'yes' + llvm: + add_deps_recursive: + tvm: + tags: _llvm diff --git a/script/build-dockerfile/customize.py b/script/build-dockerfile/customize.py index 41300cdde..d99e24392 100644 --- a/script/build-dockerfile/customize.py +++ b/script/build-dockerfile/customize.py @@ -55,7 +55,7 @@ def preprocess(i): if env.get("CM_MLOPS_REPO", "") != "": cm_mlops_repo = env["CM_MLOPS_REPO"] else: - cm_mlops_repo = "mlcommons@ck" + cm_mlops_repo = "mlcommons@cm4mlops" if env.get("CM_MLOPS_REPO_BRANCH", '') != '': cm_mlops_repo_branch_string = f" --branch {env['CM_MLOPS_REPO_BRANCH']}" diff --git a/script/download-file/customize.py b/script/download-file/customize.py index 97969418c..9918de7c3 100644 --- a/script/download-file/customize.py +++ b/script/download-file/customize.py @@ -206,7 +206,7 @@ def preprocess(i): if not pre_clean: env['CM_PRE_DOWNLOAD_CMD'] = '' - if os_info['platform'] == 'windows': + if os_info['platform'] == 'windows' and env.get('CM_DOWNLOAD_CMD', '') != '': env['CM_DOWNLOAD_CMD'] = env['CM_DOWNLOAD_CMD'].replace('&', '^&').replace('|', '^|').replace('(', '^(').replace(')', '^)') if pre_clean: env['CM_PRE_DOWNLOAD_CLEAN_CMD'] = "del /Q %CM_DOWNLOAD_FILENAME%" diff --git a/script/get-cuda-devices/run.sh b/script/get-cuda-devices/run.sh index 91d0a9e1b..3d208dd6b 100644 --- a/script/get-cuda-devices/run.sh +++ b/script/get-cuda-devices/run.sh @@ -29,8 +29,7 @@ echo "" echo "Running program ..." echo "" -cd - -#${CM_TMP_CURRENT_PATH} +cd ${CM_TMP_CURRENT_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/a.out > tmp-run.out test $? -eq 0 || exit 1 diff --git a/script/get-sys-utils-cm/_cm.json b/script/get-sys-utils-cm/_cm.json deleted file mode 100644 index a496f42c9..000000000 --- a/script/get-sys-utils-cm/_cm.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "alias": "get-sys-utils-cm", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "cache": true, - "category": "Detection or installation of tools and artifacts", - "deps": [ - { - "tags": "detect,os" - } - ], - "env": { - "CM_CLEAN_DIRS": "bin", - "CM_PACKAGE_WIN_URL": "https://zenodo.org/record/6501550/files/cm-artifact-os-windows-32.zip ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1", - "CM_SUDO": "sudo" - }, - "input_mapping": { - "skip": "CM_SKIP_SYS_UTILS" - }, - "new_env_keys": [ - "+PATH" - ], - "tags": [ - "get", - "sys-utils-cm" - ], - "uid": "bc90993277e84b8e", - "variations": { - "user": { - "env": { - "CM_PYTHON_PIP_USER": "--user" - } - } - }, - "warnings": [ - "This CM script will install extra OS system utils required for CM automation workflows!" - ] -} diff --git a/script/get-sys-utils-cm/_cm.yaml b/script/get-sys-utils-cm/_cm.yaml index 555fe7c6a..1bbfac9ee 100644 --- a/script/get-sys-utils-cm/_cm.yaml +++ b/script/get-sys-utils-cm/_cm.yaml @@ -13,8 +13,9 @@ deps: env: CM_CLEAN_DIRS: bin - CM_PACKAGE_WIN_URL: https://zenodo.org/record/6501550/files/cm-artifact-os-windows-32.zip + CM_PACKAGE_WIN_URL: https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 + ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip CM_SUDO: sudo input_mapping: diff --git a/script/get-sys-utils-min/_cm.json b/script/get-sys-utils-min/_cm.json index d033f8d1c..19ce3f21a 100644 --- a/script/get-sys-utils-min/_cm.json +++ b/script/get-sys-utils-min/_cm.json @@ -8,7 +8,7 @@ ], "env": { "CM_CLEAN_DIRS": "bin", - "CM_PACKAGE_WIN_URL": "https://zenodo.org/records/10379926/files/cm-artifact-os-windows-32.zip ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip", + "CM_PACKAGE_WIN_URL": "https://zenodo.org/records/13868077/files/cm-artifact-os-windows-32.zip?download=1 ; https://www.dropbox.com/s/2y9r2mvtu8tpexk/zlib123dllx64-bin.zip?dl=1 ; https://cKnowledge.org/ai/data/xz-5.2.9-win64.zip", "CM_SUDO": "sudo" }, "new_env_keys": [ diff --git a/script/print-hello-world-py/_cm.json b/script/print-hello-world-py/_cm.json index 5e6a0629c..7bb36e4b6 100644 --- a/script/print-hello-world-py/_cm.json +++ b/script/print-hello-world-py/_cm.json @@ -7,9 +7,6 @@ { "tags": "detect,os" }, - { - "tags": "get,sys-utils-cm" - }, { "names": [ "python", diff --git a/script/print-hello-world-py/app.py b/script/print-hello-world-py/app.py new file mode 100644 index 000000000..9dfd48b33 --- /dev/null +++ b/script/print-hello-world-py/app.py @@ -0,0 +1,16 @@ +def main(): + print ('') + + # Import cmind to test break points + import cmind.utils + import os + if os.environ.get('CM_TMP_DEBUG_UID', '') == 'f52670e5f3f345a2': + cmind.utils.debug_here(__file__, port=5678, text='Debugging main.py!').breakpoint() + + print ('HELLO WORLD from Python') + + x = 1 + print (x) + +if __name__ == '__main__': + main() diff --git a/script/print-hello-world-py/code.py b/script/print-hello-world-py/code.py deleted file mode 100644 index 735a89062..000000000 --- a/script/print-hello-world-py/code.py +++ /dev/null @@ -1,6 +0,0 @@ -def main(): - print ('') - print ('HELLO WORLD from Python') - -if __name__ == '__main__': - main() diff --git a/script/print-hello-world-py/customize.py b/script/print-hello-world-py/customize.py new file mode 100644 index 000000000..830869367 --- /dev/null +++ b/script/print-hello-world-py/customize.py @@ -0,0 +1,19 @@ +# Developer(s): Grigori Fursin + +import os + +def preprocess(i): + + os_info = i['os_info'] + env = i['env'] + meta = i['meta'] + + return {'return':0} + + +def postprocess(i): + + env = i['env'] + state = i['state'] + + return {'return':0} diff --git a/script/print-hello-world-py/run.bat b/script/print-hello-world-py/run.bat index d1881d3a3..c0980c59b 100644 --- a/script/print-hello-world-py/run.bat +++ b/script/print-hello-world-py/run.bat @@ -1,8 +1,8 @@ IF NOT DEFINED CM_TMP_CURRENT_SCRIPT_PATH SET CM_TMP_CURRENT_SCRIPT_PATH=%CD% -%CM_PYTHON_BIN_WITH_PATH% --version +rem %CM_PYTHON_BIN_WITH_PATH% --version -%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\code.py +%CM_PYTHON_BIN_WITH_PATH% %CM_TMP_CURRENT_SCRIPT_PATH%\app.py IF %ERRORLEVEL% NEQ 0 EXIT %ERRORLEVEL% echo CM_NEW_VAR_FROM_RUN=XYZ > tmp-run-env.out diff --git a/script/print-hello-world-py/run.sh b/script/print-hello-world-py/run.sh index b7c69c790..bc7e2c301 100644 --- a/script/print-hello-world-py/run.sh +++ b/script/print-hello-world-py/run.sh @@ -2,10 +2,10 @@ CM_TMP_CURRENT_SCRIPT_PATH=${CM_TMP_CURRENT_SCRIPT_PATH:-$PWD} -which ${CM_PYTHON_BIN_WITH_PATH} -${CM_PYTHON_BIN_WITH_PATH} --version +#which ${CM_PYTHON_BIN_WITH_PATH} +#${CM_PYTHON_BIN_WITH_PATH} --version -${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/code.py +${CM_PYTHON_BIN_WITH_PATH} ${CM_TMP_CURRENT_SCRIPT_PATH}/app.py test $? -eq 0 || exit $? echo "CM_NEW_VAR_FROM_RUN=$MLPERF_XYZ" > tmp-run-env.out diff --git a/script/print-hello-world/_cm.json b/script/print-hello-world/_cm.json deleted file mode 100644 index c8ccffe53..000000000 --- a/script/print-hello-world/_cm.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "alias": "print-hello-world", - "automation_alias": "script", - "automation_uid": "5b4e0237da074764", - "category": "Tests", - "default_env": { - "CM_ENV_TEST1": "TEST1" - }, - "env": { - "CM_ENV_TEST2": "TEST2" - }, - "input_mapping": { - "test1": "CM_ENV_TEST1" - }, - "new_env_keys": [ - "CM_ENV_TEST*" - ], - "new_state_keys": [ - "hello_world*" - ], - "tags": [ - "print", - "hello-world", - "hello world", - "hello", - "world", - "native-script", - "native", - "script" - ], - "uid": "b9f0acba4aca4baa" -} diff --git a/script/test-debug/_demo.py b/script/test-debug/_demo.py index 781bed321..878249a74 100644 --- a/script/test-debug/_demo.py +++ b/script/test-debug/_demo.py @@ -5,5 +5,5 @@ print(sys.executable) -r = cmind.access('run "cm-debug"') +r = cmind.access('run script "test cm-debug"') print(r) diff --git a/script/test-dummy/README-extra.md b/script/test-dummy/README-extra.md new file mode 100644 index 000000000..582991f6d --- /dev/null +++ b/script/test-dummy/README-extra.md @@ -0,0 +1 @@ +# CM script diff --git a/script/test-dummy/_cm.yaml b/script/test-dummy/_cm.yaml new file mode 100644 index 000000000..09ba01f92 --- /dev/null +++ b/script/test-dummy/_cm.yaml @@ -0,0 +1,11 @@ +alias: test-dummy +uid: 3ef5d69f929349bc + +automation_alias: script +automation_uid: 5b4e0237da074764 + +cache: false + +tags: +- test +- dummy diff --git a/script/test-dummy/customize.py b/script/test-dummy/customize.py new file mode 100644 index 000000000..d12f9b3e1 --- /dev/null +++ b/script/test-dummy/customize.py @@ -0,0 +1,22 @@ +from cmind import utils +import os + +def preprocess(i): + + os_info = i['os_info'] + + env = i['env'] + + meta = i['meta'] + + automation = i['automation'] + + quiet = (env.get('CM_QUIET', False) == 'yes') + + return {'return':0} + +def postprocess(i): + + env = i['env'] + + return {'return':0} diff --git a/script/test-dummy/run.bat b/script/test-dummy/run.bat new file mode 100644 index 000000000..648302ca7 --- /dev/null +++ b/script/test-dummy/run.bat @@ -0,0 +1 @@ +rem native script diff --git a/script/test-dummy/run.sh b/script/test-dummy/run.sh new file mode 100644 index 000000000..05a7907cf --- /dev/null +++ b/script/test-dummy/run.sh @@ -0,0 +1,2 @@ +#!/bin/bash + diff --git a/setup.py b/setup.py index e7c96a97c..2ff411286 100644 --- a/setup.py +++ b/setup.py @@ -1,147 +1,147 @@ -# setup.py -from setuptools import setup -from setuptools._distutils.dist import Distribution -from setuptools.command.install import install -import subprocess -import sys -import importlib.util -import platform -import os - -# Try to use importlib.metadata for Python 3.8+ -try: - if sys.version_info >= (3, 8): - from importlib.metadata import version, PackageNotFoundError - else: - # Fallback to pkg_resources for Python < 3.8 - import pkg_resources - PackageNotFoundError = pkg_resources.DistributionNotFound -except ImportError: - # If importlib.metadata is unavailable, fall back to pkg_resources - import pkg_resources - PackageNotFoundError = pkg_resources.DistributionNotFound - - - -class CustomInstallCommand(install): - def run(self): - self.get_sys_platform() - self.install_system_packages() - - # Call the standard run method - install.run(self) - - # Call the custom function - return self.custom_function() - - def is_package_installed(self, package_name): - try: - if sys.version_info >= (3, 8): - version(package_name) # Tries to get the version of the package - else: - pkg_resources.get_distribution(package_name) # Fallback for < 3.8 - return True - except PackageNotFoundError: - return False - - def install_system_packages(self): - # List of packages to install via system package manager - packages = [] - - git_status = self.command_exists('git') - if not git_status: - packages.append("git") - wget_status = self.command_exists('wget') - if not wget_status: - packages.append("wget") - curl_status = self.command_exists('curl') - if not curl_status: - packages.append("curl") - - name='venv' - - if name in sys.modules: - pass #nothing needed - elif self.is_package_installed(name): - pass - else: - packages.append("python3-venv") - - if packages: - if self.system == 'Linux' or self.system == 'Darwin': - manager, details = self.get_package_manager_details() - if manager: - if manager == "apt-get": - subprocess.check_call(['sudo', 'apt-get', 'update']) - subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) - elif self.system == 'Windows': - print(f"Please install the following packages manually: {packages}") - - - - def detect_package_manager(self): - package_managers = { - 'apt-get': '/usr/bin/apt-get', - 'yum': '/usr/bin/yum', - 'dnf': '/usr/bin/dnf', - 'pacman': '/usr/bin/pacman', - 'zypper': '/usr/bin/zypper', - 'brew': '/usr/local/bin/brew' - } - - for name, path in package_managers.items(): - if os.path.exists(path): - return name - - return None - - def get_package_manager_details(self): - manager = self.detect_package_manager() - if manager: - try: - version_output = subprocess.check_output([manager, '--version'], stderr=subprocess.STDOUT).decode('utf-8') - return manager, version_output.split('\n')[0] - except subprocess.CalledProcessError: - return manager, 'Version information not available' - else: - return None, 'No supported package manager found' - - # Checks if command exists(for installing required packages). - # If the command exists, which returns 0, making the function return True. - # If the command does not exist, which returns a non-zero value, making the function return False. - # NOTE: The standard output and standard error streams are redirected to PIPES so that it could be captured in future if needed. - def command_exists(self, command): - if self.system == "Linux" or self.system == 'Darwin': - return subprocess.call(['which', command], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 - elif self.system == "Windows": - return subprocess.call([command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0 - - def custom_function(self): - import cmind - #r = cmind.access({'action':'rm', 'automation':'repo', 'data_uoa':'mlcommons@cm4mlops', 'force': True}) - r = cmind.access({'action':'pull', 'automation':'repo', 'artifact':'mlcommons@cm4mlops', 'branch': 'mlperf-inference'}) - print(r) - if r['return'] > 0: - return r['return'] - - def get_sys_platform(self): - self.system = platform.system() - -setup( - name='cm4mlops', - version='0.1', - long_description='CM automations and scripts for MLOps', - long_description_content_type='text/x-rst', - packages=[], - install_requires=[ - "setuptools>=60", - "wheel", - "cmind", - "giturlparse", - "requests", - "pyyaml" - ], - cmdclass={ - 'install': CustomInstallCommand, - }, -) +# setup.py +from setuptools import setup +from setuptools._distutils.dist import Distribution +from setuptools.command.install import install +import subprocess +import sys +import importlib.util +import platform +import os + +# Try to use importlib.metadata for Python 3.8+ +try: + if sys.version_info >= (3, 8): + from importlib.metadata import version, PackageNotFoundError + else: + # Fallback to pkg_resources for Python < 3.8 + import pkg_resources + PackageNotFoundError = pkg_resources.DistributionNotFound +except ImportError: + # If importlib.metadata is unavailable, fall back to pkg_resources + import pkg_resources + PackageNotFoundError = pkg_resources.DistributionNotFound + + + +class CustomInstallCommand(install): + def run(self): + self.get_sys_platform() + self.install_system_packages() + + # Call the standard run method + install.run(self) + + # Call the custom function + return self.custom_function() + + def is_package_installed(self, package_name): + try: + if sys.version_info >= (3, 8): + version(package_name) # Tries to get the version of the package + else: + pkg_resources.get_distribution(package_name) # Fallback for < 3.8 + return True + except PackageNotFoundError: + return False + + def install_system_packages(self): + # List of packages to install via system package manager + packages = [] + + git_status = self.command_exists('git') + if not git_status: + packages.append("git") + wget_status = self.command_exists('wget') + if not wget_status: + packages.append("wget") + curl_status = self.command_exists('curl') + if not curl_status: + packages.append("curl") + + name='venv' + + if name in sys.modules: + pass #nothing needed + elif self.is_package_installed(name): + pass + else: + packages.append("python3-venv") + + if packages: + if self.system == 'Linux' or self.system == 'Darwin': + manager, details = self.get_package_manager_details() + if manager: + if manager == "apt-get": + subprocess.check_call(['sudo', 'apt-get', 'update']) + subprocess.check_call(['sudo', 'apt-get', 'install', '-y'] + packages) + elif self.system == 'Windows': + print(f"Please install the following packages manually: {packages}") + + + + def detect_package_manager(self): + package_managers = { + 'apt-get': '/usr/bin/apt-get', + 'yum': '/usr/bin/yum', + 'dnf': '/usr/bin/dnf', + 'pacman': '/usr/bin/pacman', + 'zypper': '/usr/bin/zypper', + 'brew': '/usr/local/bin/brew' + } + + for name, path in package_managers.items(): + if os.path.exists(path): + return name + + return None + + def get_package_manager_details(self): + manager = self.detect_package_manager() + if manager: + try: + version_output = subprocess.check_output([manager, '--version'], stderr=subprocess.STDOUT).decode('utf-8') + return manager, version_output.split('\n')[0] + except subprocess.CalledProcessError: + return manager, 'Version information not available' + else: + return None, 'No supported package manager found' + + # Checks if command exists(for installing required packages). + # If the command exists, which returns 0, making the function return True. + # If the command does not exist, which returns a non-zero value, making the function return False. + # NOTE: The standard output and standard error streams are redirected to PIPES so that it could be captured in future if needed. + def command_exists(self, command): + if self.system == "Linux" or self.system == 'Darwin': + return subprocess.call(['which', command], stdout=subprocess.PIPE, stderr=subprocess.PIPE) == 0 + elif self.system == "Windows": + return subprocess.call([command, '--version'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True) == 0 + + def custom_function(self): + import cmind + #r = cmind.access({'action':'rm', 'automation':'repo', 'data_uoa':'mlcommons@cm4mlops', 'force': True}) + r = cmind.access({'action':'pull', 'automation':'repo', 'artifact':'mlcommons@cm4mlops', 'branch': 'mlperf-inference'}) + print(r) + if r['return'] > 0: + return r['return'] + + def get_sys_platform(self): + self.system = platform.system() + +setup( + name='cm4mlops', + version='0.1', + long_description='CM automations and scripts for MLOps', + long_description_content_type='text/x-rst', + packages=[], + install_requires=[ + "setuptools>=60", + "wheel", + "cmind", + "giturlparse", + "requests", + "pyyaml" + ], + cmdclass={ + 'install': CustomInstallCommand, + }, +)