Sync with dev (new)

LICENSE.third-party.md

@@ -0,0 +1 @@
+This CM repository may contain CM scripts with third-party files licensed under Apache2, BSD or MIT license.

automation/cfg/README-extra.md

@@ -0,0 +1,8 @@
+Examples:
+
+```bash
+cm set cfg default 
+cm set cfg default --key.script.silent
+cm set cfg default --key.script.silent-
+
+```

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile

@@ -0,0 +1,34 @@
+FROM ubuntu:23.04
+
+# Automatically generated by the CM workflow automation meta-framework
+# https://github.com/mlcommons/ck
+
+LABEL github=""
+LABEL maintainer=""
+LABEL license=""
+
+SHELL ["/bin/bash", "-c"]
+
+ARG CM_GH_TOKEN
+ARG CM_ADD_DOCKER_GROUP_ID=""
+
+# Notes: https://runnable.com/blog/9-common-dockerfile-mistakes
+# Install system dependencies
+RUN apt-get update -y
+RUN apt-get install -y python3 python3-pip git sudo wget
+
+# Setup docker environment
+ENTRYPOINT ["/bin/bash", "-c"]
+ENV TZ="US/Pacific"
+ENV PATH="${PATH}:/home/cmuser/.local/bin"
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ >/etc/timezone
+
+# Setup docker user
+RUN groupadd ${CM_ADD_DOCKER_GROUP_ID} cm
+RUN useradd -g cm --create-home --shell /bin/bash cmuser
+RUN echo "cmuser ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+USER cmuser:cm
+WORKDIR /home/cmuser
+
+# Install python packages
+RUN python3 -m pip install --user cmind requests giturlparse tabulate --break-system-packages 

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.bat

@@ -0,0 +1 @@
+docker build -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" .

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.build.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker build -f "ubuntu-23.04.Dockerfile" -t "cknowledge/test-cm-script:ubuntu-23.04-cm-dev" .

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.bat

@@ -0,0 +1 @@
+docker run -it --entrypoint "" cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash

automation/script/docker_repro_example/ubuntu-23.04.Dockerfile.run.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+docker run -it --entrypoint "" cknowledge/test-cm-script:ubuntu-23.04-cm-dev bash

cfg/docker-basic-configurations/_cm.yaml

@@ -0,0 +1,10 @@
+alias: docker-basic-configurations
+uid: d2a0c5bb17664c93
+
+automation_alias: cfg
+automation_uid: 88dce9c160324c5d
+
+tags:
+- docker
+- basic
+- configurations

cfg/docker-basic-configurations/basic-archlinux.yaml

@@ -0,0 +1,9 @@
+uid: 9960e9fb3cb24cb3
+
+name: "Basic ArchLinux"
+
+input:
+ docker_base_image: 'archlinux'
+ docker_os: arch
+ docker_os_version: 'latest'
+

cfg/docker-basic-configurations/basic-rhel-9.yaml

@@ -0,0 +1,9 @@
+uid: 27b4afcdd8e042e8
+
+name: "Basic RHEL 9"
+
+input:
+ docker_base_image: 'registry.access.redhat.com/ubi9'
+ docker_os: 'rhel'
+ docker_os_version: '9'
+

cfg/docker-basic-configurations/basic-ubuntu-20.04.yaml

@@ -0,0 +1,9 @@
+uid: 59311e6098c14b21
+
+name: "Basic Ubuntu 20.04"
+
+input:
+ docker_base_image: 'ubuntu:20.04'
+ docker_os: ubuntu
+ docker_os_version: '20.04'
+

cfg/docker-basic-configurations/basic-ubuntu-22.04.yaml

@@ -0,0 +1,9 @@
+uid: 614aa48d90724835
+
+name: "Basic Ubuntu 22.04"
+
+input:
+ docker_base_image: 'ubuntu:22.04'
+ docker_os: ubuntu
+ docker_os_version: '22.04'
+

cfg/docker-basic-configurations/basic-ubuntu-23.04.yaml

@@ -0,0 +1,9 @@
+uid: 276bd8ab39324f5f
+
+name: "Basic Ubuntu 23.04"
+
+input:
+ docker_base_image: 'ubuntu:23.04'
+ docker_os: ubuntu
+ docker_os_version: '23.04'
+

script/app-image-corner-detection/_cm.yaml

@@ -0,0 +1,32 @@
+alias: app-image-corner-detection
+uid: 998ffee0bc534d0a
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+category: Modular application pipeline
+
+deps:
+- tags: detect,os
+
+- tags: detect,cpu
+
+- tags: download,file,_url.https://cKnowledge.org/ai/data/data.pgm
+ md5sum: 0af279e557a8de252d7ff0751a999379
+ force_cache: false
+
+
+posthook_deps:
+- skip_if_env:
+ CM_SKIP_COMPILE:
+ - 'on'
+ tags: compile,cpp-program
+- skip_if_env:
+ CM_SKIP_RUN:
+ - 'on'
+ tags: benchmark-program
+
+tags:
+- app
+- image
+- corner-detection

script/benchmark-program/README-extra.md

@@ -0,0 +1,3 @@
+This is a universal script to run and profile programs.
+
+It is a part of our universal benchmarking and optimization roadmap: https://github.com/mlcommons/cm4mlops/issues/23

script/build-docker-image/_cm.yaml

@@ -0,0 +1,46 @@
+alias: build-docker-image
+uid: 2c3c4ba2413442e7
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+cache: false
+
+category: Docker automation
+
+tags:
+- build
+- docker
+- image
+- docker-image
+- dockerimage
+
+default_env:
+ CM_DOCKER_IMAGE_REPO: local
+ CM_DOCKER_IMAGE_TAG: latest
+
+input_mapping:
+ cache: CM_DOCKER_CACHE
+ cm_repo: CM_MLOPS_REPO
+ docker_os: CM_DOCKER_OS
+ docker_os_version: CM_DOCKER_OS_VERSION
+ dockerfile: CM_DOCKERFILE_WITH_PATH
+ gh_token: CM_GH_TOKEN
+ image_name: CM_DOCKER_IMAGE_NAME
+ image_repo: CM_DOCKER_IMAGE_REPO
+ image_tag: CM_DOCKER_IMAGE_TAG
+ post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS
+ pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS
+ real_run: CM_REAL_RUN
+ script_tags: CM_DOCKER_RUN_SCRIPT_TAGS
+ push_image: CM_DOCKER_PUSH_IMAGE
+
+new_env_keys:
+- CM_DOCKER_*
+
+prehook_deps:
+- enable_if_env:
+ CM_BUILD_DOCKERFILE:
+ - 'yes'
+ - '1'
+ tags: build,dockerfile

script/build-dockerfile/_cm.yaml

@@ -0,0 +1,67 @@
+alias: build-dockerfile
+uid: e66a7483230d4641
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+tags:
+- build
+- dockerfile
+
+cache: false
+
+category: Docker automation
+
+default_env:
+ CM_DOCKER_BUILD_SLIM: 'no'
+ CM_DOCKER_IMAGE_EOL: '
+
+ '
+ CM_DOCKER_OS: ubuntu
+
+input_mapping:
+ build: CM_BUILD_DOCKER_IMAGE
+ cache: CM_DOCKER_CACHE
+ cm_repo: CM_MLOPS_REPO
+ cm_repo_flags: CM_DOCKER_ADD_FLAG_TO_CM_MLOPS_REPO
+ cm_repos: CM_DOCKER_EXTRA_CM_REPOS
+ comments: CM_DOCKER_RUN_COMMENTS
+ copy_files: CM_DOCKER_COPY_FILES
+ docker_base_image: CM_DOCKER_IMAGE_BASE
+ docker_os: CM_DOCKER_OS
+ docker_os_version: CM_DOCKER_OS_VERSION
+ extra_sys_deps: CM_DOCKER_EXTRA_SYS_DEPS
+ fake_docker_deps: CM_DOCKER_FAKE_DEPS
+ fake_run_option: CM_DOCKER_FAKE_RUN_OPTION
+ file_path: CM_DOCKERFILE_WITH_PATH
+ gh_token: CM_GH_TOKEN
+ image_repo: CM_DOCKER_IMAGE_REPO
+ image_tag: CM_DOCKER_IMAGE_TAG
+ package_manager_update_cmd: CM_PACKAGE_MANAGER_UPDATE_CMD
+ pip_extra_flags: CM_DOCKER_PIP_INSTALL_EXTRA_FLAGS
+ post_file: DOCKER_IMAGE_POST_FILE
+ post_run_cmds: CM_DOCKER_POST_RUN_COMMANDS
+ pre_run_cmds: CM_DOCKER_PRE_RUN_COMMANDS
+ real_run: CM_REAL_RUN
+ run_cmd: CM_DOCKER_RUN_CMD
+ run_cmd_extra: CM_DOCKER_RUN_CMD_EXTRA
+ script_tags: CM_DOCKER_RUN_SCRIPT_TAGS
+ skip_cm_sys_upgrade: CM_DOCKER_SKIP_CM_SYS_UPGRADE
+ push_image: CM_DOCKER_PUSH_IMAGE
+
+new_env_keys:
+- CM_DOCKERFILE_*
+
+post_deps:
+- enable_if_env:
+ CM_BUILD_DOCKER_IMAGE:
+ - 'yes'
+ - '1'
+ names:
+ - build-docker-image
+ tags: build,docker,image
+
+variations:
+ slim:
+ env:
+ CM_DOCKER_BUILD_SLIM: 'yes'

script/create-custom-cache-entry/_cm.yaml

@@ -0,0 +1,27 @@
+alias: create-custom-cache-entry
+uid: 485741440fbe4236
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+tags:
+- create
+- custom
+- cache
+- entry
+
+category: CM automation
+
+cache: true
+
+input_mapping:
+ env_key: CM_CUSTOM_CACHE_ENTRY_ENV_KEY
+ env_key2: CM_CUSTOM_CACHE_ENTRY_ENV_KEY2
+ path: CM_CUSTOM_CACHE_ENTRY_PATH
+ to: CM_CUSTOM_CACHE_ENTRY_PATH
+
+new_env_keys:
+- CM_CUSTOM_CACHE_ENTRY*
+
+print_env_at_the_end:
+ CM_CUSTOM_CACHE_ENTRY_PATH: "Path to custom cache entry"

script/create-custom-cache-entry/customize.py

@@ -0,0 +1,44 @@
+from cmind import utils
+import os
+import shutil
+
+def preprocess(i):
+
+ # CM script internal variables
+ env = i['env']
+
+ extra_cache_tags = []
+ if env.get('CM_EXTRA_CACHE_TAGS','').strip() == '':
+ print ('')
+ extra_cache_tags_str = input('Enter extra tags for the custom CACHE entry separated by comma: ')
+
+ extra_cache_tags = extra_cache_tags_str.strip().split(',')
+
+ return {'return':0, 'add_extra_cache_tags':extra_cache_tags}
+
+def postprocess(i):
+
+ env = i['env']
+
+ path = env.get('CM_CUSTOM_CACHE_ENTRY_PATH','').strip()
+
+ if path!='':
+ if not os.path.isdir(path):
+ os.makedirs(path)
+ else:
+ path = os.getcwd()
+
+ x = ''
+ env_key = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY', '')
+ if env_key != '': x = env_key+'_'
+
+ env['CM_CUSTOM_CACHE_ENTRY_{}PATH'.format(x)] = path
+ env['CM_CUSTOM_CACHE_ENTRY_PATH'] = path
+
+ env_key2 = env.get('CM_CUSTOM_CACHE_ENTRY_ENV_KEY2', '')
+ v = env.get(env_key2, '')
+ real_path = v if v != '' else path
+
+ env['CM_CUSTOM_CACHE_ENTRY_{}REAL_PATH'.format(x)] = real_path
+
+ return {'return': 0}

script/generate-docs-for-all-scripts.cmd

@@ -0,0 +1 @@
+cm doc script --repos=mlcommons@cm4mlops --output_dir=..

script/get-cuda-devices/_cm.yaml

@@ -0,0 +1,41 @@
+alias: get-cuda-devices
+uid: 7a3ede4d3558427a
+
+automation_alias: script
+automation_uid: 5b4e0237da074764
+
+tags:
+- get
+- cuda-devices
+
+cache: false
+
+can_force_cache: true
+
+category: CUDA automation
+
+clean_files:
+- tmp-run.out
+
+deps:
+- names:
+ - cuda
+ tags: get,cuda,_toolkit
+
+docker:
+ run: false
+ all_gpus: 'yes'
+ skip_run_cmd: 'no'
+ skip_cm_sys_upgrade: 'yes'
+ cm_repo_flags: '--checkout=dev'
+ use_host_group_id: 'yes'
+ image_tag_extra: '-cm-dev'
+
+new_env_keys:
+- CM_CUDA_DEVICE_*
+
+new_state_keys:
+- cm_cuda_device_prop
+
+print_files_if_script_error:
+- tmp-run.out